diff --git a/.gitattributes b/.gitattributes index c7d9f3332a950355d5a77d85000f05e6f45435ea..264d7c47b65d61eda7351ac2b339be10bad281c4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -32,3 +32,75 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_3.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_4.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_0.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_1.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_0.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_5.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_2.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_2.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_0.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_5.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_3.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_4.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_3.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_2.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_2.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_2.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_4.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_0.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_5.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_0.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_4.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_3.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_2.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_2.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_1.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_1.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_5.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_3.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_0.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_1.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_1.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_3.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_5.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_0.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_5.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_4.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_1.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_3.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_1.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_3.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_5.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_5.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_2.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_2.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_1.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_3.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_4.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_0.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_1.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_4.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_0.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_5.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_4.jsonl filter=lfs diff=lfs merge=lfs -text +perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_4.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/perplexity/3510897.err b/perplexity/3510897.err new file mode 100644 index 0000000000000000000000000000000000000000..23de8a97c1d20ff8bf316a8491694a38b6376997 --- /dev/null +++ b/perplexity/3510897.err @@ -0,0 +1,3643 @@ +12: 2023-05-13 09:35:19.184552: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:19.184557: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:19.184559: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:19.184573: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:19.184573: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:19.184581: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:19.184583: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:19.184599: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: 2023-05-13 09:35:19.184774: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:19.184793: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:19.184805: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: 2023-05-13 09:35:19.184806: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:19.184829: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:19.185048: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:19.185051: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:19.185043: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: 2023-05-13 09:35:19.184724: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:19.184763: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:19.184784: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: 2023-05-13 09:35:19.184910: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:19.184920: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:19.184911: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:19.184891: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:19.184923: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:19.185086: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:19.185099: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:19.184885: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:19.184905: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:19.184925: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:19.185156: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:19.185172: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:19.185173: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:19.184825: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:19.184833: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:19.184888: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:19.184895: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:19.184926: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: 2023-05-13 09:35:19.185095: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:19.185085: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:19.184926: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:19.184926: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:19.185133: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:19.185154: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:19.185164: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: 2023-05-13 09:35:19.184810: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:19.184812: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:19.184821: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:19.185159: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:19.185191: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:19.184837: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:19.184854: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:19.184835: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:19.184928: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:19.185108: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:19.184932: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:19.184938: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:19.184955: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:19.185196: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:19.185205: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:19.185187: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:19.185203: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:19.184947: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:19.184956: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:19.185220: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:19.185228: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:19.185249: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: 2023-05-13 09:35:19.184836: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:19.184845: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:19.185232: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:19.184942: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:19.184948: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:19.184997: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:19.185246: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:19.185243: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:19.185264: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:19.185281: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:19.185290: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:19.185555: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:19.185559: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:19.185575: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: 2023-05-13 09:35:19.185292: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:19.185587: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:19.185608: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:19.185340: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:19.185619: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:19.185628: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:19.185350: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:19.185362: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:19.185372: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: 2023-05-13 09:35:19.185337: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:19.185642: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:19.185398: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:19.185413: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:19.185423: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:19.185389: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:19.185386: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:19.185434: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:19.185741: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:19.185760: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:19.185773: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: 2023-05-13 09:35:19.185551: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:19.185551: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:19.185584: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: 2023-05-13 09:35:19.185411: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:19.185413: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:19.185409: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: 2023-05-13 09:35:19.185459: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:19.185462: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:19.185469: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:19.185785: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:19.185593: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:19.185594: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:19.185486: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:19.185744: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:19.185754: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:19.185782: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: 2023-05-13 09:35:19.185801: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:19.185815: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:19.185823: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: 2023-05-13 09:35:19.185605: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:19.185606: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:19.185608: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: 2023-05-13 09:35:19.185706: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:19.185722: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:19.185752: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:19.185576: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:19.185587: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:19.185583: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: 2023-05-13 09:35:19.185645: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:19.185656: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:19.185650: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: 2023-05-13 09:35:19.185833: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:19.185675: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:19.185690: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:19.185773: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:19.185780: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:19.185784: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: 2023-05-13 09:35:19.185756: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:19.185772: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:19.185773: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: 2023-05-13 09:35:19.185678: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:19.185694: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:19.185790: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:19.185799: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:19.185785: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:19.185790: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:19.185704: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:19.185793: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:19.185786: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:19.185810: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: 2023-05-13 09:35:19.185803: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:19.185862: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:19.185545: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:19.185569: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:19.185586: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:19.185810: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:19.185872: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:19.185872: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:19.185575: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:19.185601: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:19.185606: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:19.185615: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:19.185630: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:19.185824: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:19.185836: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:19.185844: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: 2023-05-13 09:35:19.185772: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:19.185788: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:19.185788: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:19.185595: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:19.185604: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:19.186306: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:19.186326: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:19.186340: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:19.185829: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:19.185877: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:19.185790: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:19.185800: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:19.185608: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:19.185599: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:19.185602: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:19.186352: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:19.186084: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:19.186073: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:19.186099: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: 2023-05-13 09:35:19.185715: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:19.185712: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:19.185759: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: 2023-05-13 09:35:19.185886: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:19.185887: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:19.185874: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: 2023-05-13 09:35:19.185803: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:19.185816: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:19.185830: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:19.186367: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:19.186384: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:19.186377: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: 2023-05-13 09:35:19.186146: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:19.186160: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:19.186168: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:19.185775: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:19.185780: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:19.186387: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:19.186076: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:19.186102: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:19.186255: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:19.186260: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:19.186259: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:19.186187: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:19.186121: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:19.186132: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:19.186273: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:19.186278: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:19.186216: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:19.186208: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:19.186217: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: 2023-05-13 09:35:19.186135: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:19.186282: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:19.186298: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:19.186301: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:19.186221: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:19.186154: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:19.186163: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:19.186170: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:19.186673: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:19.186686: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:19.186692: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:19.186673: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:19.186698: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:19.186685: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:19.186706: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:19.186721: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: 2023-05-13 09:35:19.186638: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:19.186658: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:19.186661: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:19.186646: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:19.186672: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:19.186802: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:19.186812: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:19.186826: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: 2023-05-13 09:35:19.186694: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:19.186692: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:19.186700: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:19.186846: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:19.186854: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:19.186850: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:19.186848: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:19.186865: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:19.187278: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:19.187301: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:19.187313: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:19.187317: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:19.187337: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:19.187359: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:19.187364: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:19.187400: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:35.148130: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:35.148141: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:35.148155: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:35.148171: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:35.148178: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:35.148169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:35.148179: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:35.148189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:35.148799: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:35.148817: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:35.148823: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:35.148837: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:35.148842: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:35.148850: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:35.148852: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:35.148858: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:35:35.148446: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:35.148463: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:35.148717: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:35.148482: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:35.148478: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:35.148739: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:35.148489: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:35.148756: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:35.148675: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:35.148493: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:35.148770: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:35.148505: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:35.148782: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:35.148702: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:35.148499: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:35.148784: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:35.148704: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:35.148982: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:35.148786: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:35.148720: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:35.148999: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:35.148789: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:35.148733: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:35.149011: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:35.148734: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:35.149330: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:35.149350: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:35.148745: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:35.149371: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:35.148735: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:35.149029: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:35:35.149031: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:35.149382: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:35.149253: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:35.149268: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:35.149281: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:35:35.149038: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:35:35.149041: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:35.149396: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:35.149393: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:35.149287: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:35.149294: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:35:35.149055: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:35.149411: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:35.149300: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:35.149308: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:35.149310: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:35.149417: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:35.148860: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:35.148878: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:35.148893: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:35.148913: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:35.148932: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:35.148912: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:35.148921: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:35.148949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:35.148937: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:35.148964: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:35.148940: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:35.148972: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:35.149503: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:35.148982: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:35.149520: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:35.148983: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:35.149534: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:35.149000: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:35.148994: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:35.149478: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:35.149495: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:35.149556: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:35.149560: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:35.149509: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:35.149515: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:35.149527: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:35.149532: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:35.149542: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:35.149567: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:35.149569: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:35.149581: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:35.149548: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:35.152655: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:35.152671: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:35.152691: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:35.152705: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:35.152707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:35.152711: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:35.152690: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:35.152720: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:35.153170: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:35.153190: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:35.153194: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:35.153213: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:35.153216: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:35.153225: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:35.153232: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:35.153237: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:35.157400: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:35.157420: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:35.157462: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:35.157468: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:35.157473: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:35.157475: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:35.157486: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:35.157441: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:35.157841: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:35.157861: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:35.157872: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:35.157891: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:35.157892: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:35.157904: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:35.157906: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:35.157919: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:35:35.160125: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.160139: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.160149: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.160159: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.160166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.160168: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.160175: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.160182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.160611: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:35:35.160629: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:35:35.160639: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:35:35.160650: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:35:35.160655: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:35:35.160662: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:35:35.160666: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:35:35.160674: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:35.162163: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:35.162183: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:35.162203: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:35.162208: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:35.162211: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:35.162194: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:35.162219: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:35.162223: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:35.162623: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:35.162635: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:35.162646: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:35.162658: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:35.162663: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:35.162667: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:35.162672: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:35.162681: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.163933: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.163953: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.163966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.163981: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.163987: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.163991: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.164002: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.163998: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.164463: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.164483: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.164495: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.164513: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.164523: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.164274: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.164529: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.164537: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.164553: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.164290: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.164307: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.164316: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.164321: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.164325: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.164335: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.164336: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.164735: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.164740: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.164746: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.164768: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.164767: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.164773: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.164779: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.164786: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.164135: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.164151: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.164168: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.164171: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.164177: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.164202: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.164192: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.164188: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.164730: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.164748: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.164757: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.164759: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.164769: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.164781: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.164787: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.164791: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:35.164998: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:35.165017: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:35.165038: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:35.165062: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:35.165071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:35.165078: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:35.165082: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:35.165074: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:35.165485: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:35.165505: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:35.165531: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:35.165541: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:35.165544: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:35.165556: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:35.165443: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:35.165562: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:35.165569: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:35.165462: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:35.165473: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:35.165484: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:35.165494: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:35.165498: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:35.165502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:35.165508: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:35.165933: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:35.165952: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:35.165964: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:35.165976: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:35.165987: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:35.165990: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:35.165993: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:35.166001: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:35.166560: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:35.166577: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:35.166592: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:35.166600: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:35.166606: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:35.166607: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:35.166619: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:35.166619: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:35.167076: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:35.167095: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:35.167108: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:35.167118: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:35.167124: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:35.167126: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:35.167140: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:35.167148: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.168732: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.168755: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.168775: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.168781: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.168786: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.168755: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.168790: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.168794: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.169570: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.169591: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.169600: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.169614: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.169617: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.169624: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.169636: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.169641: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:35.170523: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:35.170545: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:35.170551: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:35.170560: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:35.170569: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:35.170572: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:35.170578: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:35.170583: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:35.170988: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:35.171015: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:35.171018: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:35.171032: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:35.171035: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:35.171040: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:35.171044: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:35.171049: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:35.171808: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:35.171828: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:35.171835: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:35.171844: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:35.171858: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:35.171862: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:35.171864: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:35.171875: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:35.172232: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:35.172260: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:35.172262: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:35.172270: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:35.172273: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:35.172276: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:35.172281: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:35.172296: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:35.172129: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:35.172146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:35.172159: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:35.172171: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:35.172170: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:35.172181: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:35.172187: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:35.172175: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:35.172630: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:35.172646: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:35.172660: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:35.172667: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:35.172676: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:35:35.172400: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:35.172678: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:35:35.172419: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:35.172689: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:35.172692: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:35:35.172431: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:35.172436: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:35.172458: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:35.172445: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:35.172465: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:35.172460: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:35.172639: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:35.172947: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:35:35.172966: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:35:35.172978: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:35:35.172984: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:35.172659: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:35.172993: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:35.172669: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:35.173001: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:35.172687: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:35.173006: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:35:35.173012: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:35.172691: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:35.172696: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:35.172707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:35.172704: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:35.173100: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:35.173114: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:35.173135: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:35.173130: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:35.173154: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:35.173157: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:35.173166: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:35.173169: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:35.173895: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:35.173915: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:35.173928: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:35.173945: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:35.173956: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:35.173930: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:35.173960: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:35.173967: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:35.174469: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:35.174490: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:35.174497: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:35.174511: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:35.174525: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:35.174530: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:35.174536: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:35.174542: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:35.174758: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:35.174549: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:35.174779: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:35.174786: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:35.174794: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:35.174567: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:35.174804: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:35.174579: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:35.174806: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:35.174591: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:35.174814: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:35.174592: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:35.174818: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:35.174606: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:35.175209: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:35.174603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:35.175218: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:35.174611: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:35.175079: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:35.175080: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:35.175233: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:35.175243: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:35.175247: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:35.175250: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:35.175252: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:35.175087: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:35.175089: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:35.175089: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:35.175096: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:35.175101: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:35.175263: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:35.175109: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:35.175077: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:35:35.175530: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:35.175099: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:35.175134: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:35.175141: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:35:35.175547: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:35.175146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:35:35.175558: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:35.175155: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:35:35.175571: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:35.175147: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:35:35.175575: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:35.175107: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:35:35.175578: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:35:35.175588: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:35.175666: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:35.175583: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:35:35.176036: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:35.176044: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:35.176059: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:35.175689: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:35.176073: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:35.175706: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:35.175716: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:35.176073: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:35.175730: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:35.176080: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:35.175743: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:35.175751: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:35.175755: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:35.176088: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:35.176093: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:35.181617: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:35.181643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:35:35.181529: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:35.181656: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:35:35.181553: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:35.181692: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:35:35.181571: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:35.181700: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:35:35.181582: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:35.181687: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:35:35.181582: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:35.181710: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:35:35.181591: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:35.181758: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:35:35.181605: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:35.182270: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:35.181608: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:35.182293: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:35.182136: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:35.182309: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:35.182156: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:35.182165: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:35.182159: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:35.182336: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:35.182340: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:35.182342: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:35.182347: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:35.182196: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:35.182198: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:35.182202: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:35.182224: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:35.182366: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:35.182152: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:35.182181: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:35.182201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:35.182224: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:35.182239: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:35.182542: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:35.182571: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:35.182603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:35.182621: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:35.182633: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:35.182642: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:35.182644: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:35.182651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:35.183208: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:35.183224: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:35.183244: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:35.183251: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:35.183262: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:35.183268: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:35.183276: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:35.183280: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:35.183642: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:35.183660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:35.183673: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:35.183694: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:35.183696: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:35.183705: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:35.183683: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:35.183707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:35.184281: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:35.184307: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:35.184336: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:35.184349: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:35.184356: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:35.184362: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:35.184369: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:35.184374: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:35.182235: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:35.182249: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:35.182254: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:35.182813: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:35.182830: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:35.182840: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:35.182854: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:35.182862: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:35.182867: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:35.182873: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:35.182887: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:36:08.387156: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.387188: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.387199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.387219: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.387239: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.387240: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.387263: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.387284: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.395759: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.395794: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.395808: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.395849: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.395857: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.395866: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.395882: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.395899: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.397954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.397978: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.398015: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.398001: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.398011: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.398020: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.398025: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.398039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.398042: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.398047: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.398058: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.398083: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.398085: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.398096: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.398106: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.398132: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398357: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398387: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398405: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398431: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398460: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398467: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398475: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398488: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.398962: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.398992: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.398888: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.399003: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.399292: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.399023: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.399032: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.399326: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.399034: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.399336: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.398913: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.399049: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.399354: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.398934: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.399169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.399209: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.399361: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.398944: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.399193: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.399368: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.398954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.399203: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.399386: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.398961: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.399225: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.399438: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.398974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.399231: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.398979: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.399233: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.399239: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.399241: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.399338: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.399286: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.399354: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.399371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.399303: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.399378: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.399324: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.399385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.399328: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.399388: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.399347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.399388: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.399351: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.399404: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.399358: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.399399: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.400994: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.401010: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.401149: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.401150: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.400997: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.401150: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.401180: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.400997: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.401150: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.400998: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.401155: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.400999: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.401155: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.401197: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.401001: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.401153: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.401216: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.401006: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.401158: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.401170: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.401226: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.401006: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.401026: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.401174: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.401174: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.401175: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.401235: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.401027: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:08.401028: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:08.401027: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.401179: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.401176: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.401177: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.401240: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.401030: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:08.401028: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:08.401029: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.401179: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.401254: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.401268: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.401454: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.401466: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.401482: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.401509: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.401517: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.401518: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.401727: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.401518: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.401534: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.401747: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.401749: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.401558: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.401763: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.401769: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.401778: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.401594: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.401782: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.401634: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.401788: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.401844: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.401647: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.401678: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.401680: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.401687: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.401883: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.401701: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.401895: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.401899: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.401904: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.401916: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.401923: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.401921: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.402071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.401768: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.402095: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402098: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.402091: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.401819: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.402105: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.401775: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.401848: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.402111: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.401775: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.401861: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.402114: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.401778: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.401873: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402098: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.402120: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.401780: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.401785: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.401875: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402097: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.402240: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.401780: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.401882: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402101: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.401923: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.401788: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.401791: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.401793: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.401880: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402104: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.401796: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.401798: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.401891: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402104: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402120: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.401798: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.401945: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402119: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:08.402121: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:08.402121: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.401805: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.401814: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:08.402116: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402123: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.401930: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.401823: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402122: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:08.402141: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.401953: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.401951: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402174: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402194: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.401955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402135: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.401969: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.401966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.401985: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401952: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.402254: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.401977: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.402002: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.401982: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402137: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.401992: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.401990: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402131: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.402009: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401960: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.402256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402137: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.402027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401974: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.402260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402143: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401974: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401974: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401978: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.402260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.402432: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402144: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401978: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401980: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.402263: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402143: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401984: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.402003: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.402266: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402155: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401997: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.402019: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.402266: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402155: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.402158: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.402157: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.402267: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.402282: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.402162: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.402163: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.402164: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402163: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.402283: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.402286: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.402287: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.402449: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402165: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.402288: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.402284: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.402289: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.402462: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.402289: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.402484: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402164: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.402491: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.402488: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.402501: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402170: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402173: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.402363: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402174: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.402649: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402175: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402196: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402196: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402197: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402198: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.402367: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402196: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402199: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402200: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.402365: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402199: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.402365: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.402370: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.402367: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.402370: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.402374: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.402381: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.402384: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.402385: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.402386: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.402388: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.402388: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.402390: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.402390: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.402618: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.402620: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.402620: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.402624: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.402627: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.402634: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.402634: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.402627: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.402627: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.402629: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.402642: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.402641: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.402647: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.402649: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.402649: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.402650: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.402691: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.402708: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.403088: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.402692: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.402713: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.403087: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.402692: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.402715: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.403088: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.402694: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.402712: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.403089: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.402698: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.402716: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.403092: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.402700: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.402716: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.403098: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.402699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.402718: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.403099: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.402701: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.402712: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.402720: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.402727: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:08.403103: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.403112: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.402713: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.402713: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.402713: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.402728: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.402728: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.402732: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:08.403112: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:08.403114: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:08.403114: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.402714: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.402717: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.402719: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.402734: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.402734: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.402735: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:08.403113: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:08.403118: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:08.403119: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.402720: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.402736: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:08.403119: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.403154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.403157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.403158: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.403200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.403157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.403163: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.403166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.403201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.403168: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.403202: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.403170: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.403174: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.403444: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.403206: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.403173: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.403174: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.403179: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:08.403206: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.403218: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403118: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.403177: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.403182: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.403184: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.403447: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.403211: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.403185: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.403447: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.403213: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.403218: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403120: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.403450: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.403213: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.403225: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403121: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.403450: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.403225: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:08.403226: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:08.403231: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403121: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.403450: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.403231: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:08.403233: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403123: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.403453: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.403461: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403122: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.403455: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.403466: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403122: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.403466: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.403470: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.403469: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403123: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.403139: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.403472: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.403474: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.403473: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403140: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403142: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403141: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403143: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403144: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403145: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403145: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.403258: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.403260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.403260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.403262: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.403264: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.403267: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.403269: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.403352: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.403274: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.403277: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.403321: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.403277: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.403278: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.403278: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.403280: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.403284: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.403287: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.403354: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.403281: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.403357: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.403322: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.403358: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.403326: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.403361: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.403328: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.403360: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.403325: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.403363: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.403327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.403367: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.403385: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.403331: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.403385: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.403386: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.403387: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.403330: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.403340: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.403387: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.403389: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.403389: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.403340: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.403344: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.403344: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.403389: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.403345: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.403344: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.403348: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.403350: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.403902: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.403903: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.403902: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.403906: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.403902: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.403908: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.403908: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.403919: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.403909: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.403924: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.403925: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.403924: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.403926: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.403927: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.403926: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.403933: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.408080: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.408105: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.408123: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.408141: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.408154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.408169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.408177: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.408973: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.409009: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.409027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.409034: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.409049: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.409053: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.409069: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.409251: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.410165: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.410193: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.410211: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.410226: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.410243: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.410252: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.410259: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.410433: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.410268: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.410269: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.410271: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.410277: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.410282: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.410287: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.410289: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.410289: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.410285: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.410292: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.410287: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.410289: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.410302: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.410309: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.410312: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.410313: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.411259: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.411262: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.411262: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.411264: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.411268: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.411268: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.411271: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.411271: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.411277: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.411278: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.411278: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.411281: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.411283: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.411286: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.411285: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.411285: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.411974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.412007: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.412040: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.412069: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.412070: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.412073: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.412086: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.412270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.413192: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.413197: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.413198: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.413200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.413209: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.413211: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.413204: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.413203: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.413205: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.413214: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.413205: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.413217: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.413223: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.413225: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.413226: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.413227: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.417134: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.417164: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.417196: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.417229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.417251: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.417260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.417471: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.417493: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.418423: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.424821: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.424846: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.424874: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.424885: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.424895: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.424911: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.425097: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.425102: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.426154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.426158: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.426161: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.426165: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.426171: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.426173: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.426160: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.426164: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.426161: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.426167: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.426185: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.426185: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.426188: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.426190: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.426190: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.426191: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.441395: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.441441: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.441451: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.441479: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.441487: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.441493: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.441514: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.441536: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442535: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442549: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.442545: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442547: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442547: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442548: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442552: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442554: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442569: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.442557: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442573: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.442574: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.442574: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.442575: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.442575: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.442579: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.444404: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.444442: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.444455: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.444496: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.444499: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.444507: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.444508: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.444700: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.445724: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.445738: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.445847: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.445733: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445880: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.445737: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445889: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.445736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.445736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445933: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.445739: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445945: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.445743: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445968: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.445748: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.445762: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.446127: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.445764: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.445766: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.445766: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.445765: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.445767: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.445771: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.446579: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.446621: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.446629: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.446657: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.446679: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.446674: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.446702: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.446887: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.446465: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.446489: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.446526: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.446523: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.446539: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.446548: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.446555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.446745: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.447273: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.447278: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.447279: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.447282: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.447292: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.447294: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.447284: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.447287: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.447286: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.447296: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.447297: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.447288: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.447310: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.447312: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.447314: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.447316: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.448029: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.447811: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.447814: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.447826: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.448035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.447827: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.447824: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.448039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.448055: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.447824: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.448057: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.448054: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.447830: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.448045: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.447826: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.448057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.447823: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.448057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.448071: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.447829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.447847: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.448079: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.448080: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.447847: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.447848: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.447849: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.448104: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.447850: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.447850: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.448111: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.448120: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.448126: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.408305: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.409270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.409270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.409270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.409271: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.409276: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.409277: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.409277: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.409282: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.409289: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.409289: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.409290: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.409292: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.409293: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.409295: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.409295: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.409297: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.418424: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.418424: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.418427: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.418428: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.418430: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.418434: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.418430: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.418446: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.418448: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.418448: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.418450: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.418450: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.418451: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.418453: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.418454: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module scaled_upper_triang_masked_softmax_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module scaled_upper_triang_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module scaled_masked_softmax_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module scaled_masked_softmax_cuda... + 0: Loading extension module fused_mix_prec_layer_norm_cuda... + 4: Successfully preprocessed all matching files. + 4: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. + 7: Successfully preprocessed all matching files. +15: Successfully preprocessed all matching files. +15: Successfully preprocessed all matching files. +15: Successfully preprocessed all matching files. +20: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. +19: Successfully preprocessed all matching files. +23: Successfully preprocessed all matching files. + 6: Successfully preprocessed all matching files. + 6: Successfully preprocessed all matching files. +27: Successfully preprocessed all matching files. +27: Successfully preprocessed all matching files. +30: Successfully preprocessed all matching files. +21: Successfully preprocessed all matching files. +21: Successfully preprocessed all matching files. +11: Successfully preprocessed all matching files. +11: Successfully preprocessed all matching files. +11: Successfully preprocessed all matching files. +23: Successfully preprocessed all matching files. + 4: Successfully preprocessed all matching files. + 4: Successfully preprocessed all matching files. + 8: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. +17: Successfully preprocessed all matching files. +17: Successfully preprocessed all matching files. + 4: Successfully preprocessed all matching files. +12: Successfully preprocessed all matching files. +12: Successfully preprocessed all matching files. +12: Successfully preprocessed all matching files. +17: Successfully preprocessed all matching files. +17: Successfully preprocessed all matching files. + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: + 2: + 2: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: + 3: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: + 9: + 9: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +10: +10: +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: +13: +13: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +20: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +21: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: +14: +14: +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: +16: +16: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +18: +18: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +22: +22: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: +24: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: +25: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: +26: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +28: +28: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: +29: +29: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... + 0: Building extension module utils... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... + 1: Building extension module utils... + 1: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 1: Loading extension module utils... + 0: Loading extension module utils... + 1: Loading extension module utils... + 1: Loading extension module utils... + 1: Loading extension module utils... + 6: Loading extension module utils... + 6: Loading extension module utils... + 6: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 3: Loading extension module utils... + 3: Loading extension module utils... + 3: Loading extension module utils... + 3: Loading extension module utils... + 5: Loading extension module utils... + 5: Loading extension module utils... + 5: Loading extension module utils... + 5: Loading extension module utils... + 4: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... + 4: Loading extension module utils... +16: Loading extension module utils... +22: Loading extension module utils... +23: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... +20: Loading extension module utils... +20: Loading extension module utils... +20: Loading extension module utils... +20: Loading extension module utils... +15: Loading extension module utils... +16: Loading extension module utils... +10: Loading extension module utils... +22: Loading extension module utils... +23: Loading extension module utils... +16: Loading extension module utils... +22: Loading extension module utils... +23: Loading extension module utils... +22: Loading extension module utils... +23: Loading extension module utils... +15: Loading extension module utils... + 8: Loading extension module utils... +12: Loading extension module utils... + 8: Loading extension module utils... +12: Loading extension module utils... +15: Loading extension module utils... +13: Loading extension module utils... +13: Loading extension module utils... + 8: Loading extension module utils... +12: Loading extension module utils... +15: Loading extension module utils... + 8: Loading extension module utils... +12: Loading extension module utils... +13: Loading extension module utils... +24: Loading extension module utils... + 7: Loading extension module utils... +17: Loading extension module utils... +24: Loading extension module utils... +19: Loading extension module utils... + 7: Loading extension module utils... +13: Loading extension module utils... +30: Loading extension module utils... +25: Loading extension module utils... +17: Loading extension module utils... +24: Loading extension module utils... +19: Loading extension module utils... +11: Loading extension module utils... +27: Loading extension module utils... +19: Loading extension module utils... +30: Loading extension module utils... +14: Loading extension module utils... +27: Loading extension module utils... +17: Loading extension module utils... +25: Loading extension module utils... + 7: Loading extension module utils... +24: Loading extension module utils... +30: Loading extension module utils... +11: Loading extension module utils... +19: Loading extension module utils... +26: Loading extension module utils... +27: Loading extension module utils... +17: Loading extension module utils... +31: Loading extension module utils... +14: Loading extension module utils... +11: Loading extension module utils... +30: Loading extension module utils... +25: Loading extension module utils... +28: Loading extension module utils... +31: Loading extension module utils... + 7: Loading extension module utils... +26: Loading extension module utils... +14: Loading extension module utils... +27: Loading extension module utils... +11: Loading extension module utils... +28: Loading extension module utils... +25: Loading extension module utils... +26: Loading extension module utils... +31: Loading extension module utils... +14: Loading extension module utils... +28: Loading extension module utils... +26: Loading extension module utils... +31: Loading extension module utils... +28: Loading extension module utils... +16: Loading extension module utils... +29: Loading extension module utils... +10: Loading extension module utils... +21: Loading extension module utils... +10: Loading extension module utils... +21: Loading extension module utils... +10: Loading extension module utils... +21: Loading extension module utils... + 0: Loading extension module utils...Loading extension module utils...Loading extension module utils... + 0: + 0: +21: Loading extension module utils... +29: Loading extension module utils... +29: Loading extension module utils... +29: Loading extension module utils... + 0: Loading extension module utils...Loading extension module utils...Loading extension module utils... + 0: + 0: + 1: Loading extension module utils... + 6: Loading extension module utils... + 1: Loading extension module utils... + 6: Loading extension module utils... + 1: Loading extension module utils... + 6: Loading extension module utils... + 1: Loading extension module utils... + 6: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... + 7: Loading extension module utils... + 7: Loading extension module utils... + 7: Loading extension module utils... + 7: Loading extension module utils... +15: Loading extension module utils... +11: Loading extension module utils... +15: Loading extension module utils... + 5: Loading extension module utils... + 9: Loading extension module utils... +15: Loading extension module utils... + 5: Loading extension module utils... +11: Loading extension module utils... + 9: Loading extension module utils... + 9: Loading extension module utils... +15: Loading extension module utils... + 5: Loading extension module utils... +23: Loading extension module utils... +11: Loading extension module utils... + 8: Loading extension module utils... + 5: Loading extension module utils... +23: Loading extension module utils... +11: Loading extension module utils... + 8: Loading extension module utils... + 3: Loading extension module utils... +17: Loading extension module utils... + 8: Loading extension module utils... +23: Loading extension module utils... +17: Loading extension module utils... + 3: Loading extension module utils... +23: Loading extension module utils... + 8: Loading extension module utils... +17: Loading extension module utils... +18: Loading extension module utils... +14: Loading extension module utils... + 3: Loading extension module utils... +13: Loading extension module utils... +17: Loading extension module utils... +18: Loading extension module utils... + 3: Loading extension module utils... + 9: Loading extension module utils... +31: Loading extension module utils... +14: Loading extension module utils... +12: Loading extension module utils... + 9: Loading extension module utils... +13: Loading extension module utils... +16: Loading extension module utils... +18: Loading extension module utils... +30: Loading extension module utils... +14: Loading extension module utils... +20: Loading extension module utils... +31: Loading extension module utils... + 2: Loading extension module utils... +13: Loading extension module utils... + 9: Loading extension module utils... +18: Loading extension module utils... +19: Loading extension module utils... +12: Loading extension module utils... +30: Loading extension module utils... +10: Loading extension module utils... +13: Loading extension module utils... +31: Loading extension module utils... +21: Loading extension module utils... + 2: Loading extension module utils... +16: Loading extension module utils... +14: Loading extension module utils... +19: Loading extension module utils... + 9: Loading extension module utils... +20: Loading extension module utils... +30: Loading extension module utils... +22: Loading extension module utils... +12: Loading extension module utils... +27: Loading extension module utils... +21: Loading extension module utils... +31: Loading extension module utils... +16: Loading extension module utils... +10: Loading extension module utils... +19: Loading extension module utils... + 2: Loading extension module utils... +12: Loading extension module utils... +22: Loading extension module utils... +20: Loading extension module utils... +30: Loading extension module utils... + 2: Loading extension module utils... +27: Loading extension module utils... +21: Loading extension module utils... +10: Loading extension module utils... +19: Loading extension module utils... +16: Loading extension module utils... +22: Loading extension module utils... +25: Loading extension module utils... +10: Loading extension module utils... +27: Loading extension module utils... +20: Loading extension module utils... +26: Loading extension module utils... +21: Loading extension module utils... +22: Loading extension module utils... +27: Loading extension module utils... +25: Loading extension module utils... +26: Loading extension module utils... +24: Loading extension module utils... +25: Loading extension module utils... +24: Loading extension module utils... +26: Loading extension module utils... +25: Loading extension module utils... +28: Loading extension module utils... +24: Loading extension module utils... +26: Loading extension module utils... +24: Loading extension module utils... +28: Loading extension module utils... +29: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +29: Loading extension module utils... + 9: Loading extension module utils... +29: Loading extension module utils... +29: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 0: + 0: Loading extension module utils... + 0: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 1: + 1: + 1: Loading extension module utils...Loading extension module utils... + 1: Loading extension module utils... + 1: + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 6: + 6: + 6: Loading extension module utils... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: +19: Loading extension module utils... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +12: +12: Loading extension module utils...Loading extension module utils... +12: +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +15: +15: +15: Loading extension module utils...Loading extension module utils... +15: +15: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +11: +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 4: + 4: Loading extension module utils...Loading extension module utils... + 4: + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +21: +21: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +23: +23: Loading extension module utils... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +23: +23: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... +23: +23: +23: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +23: +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: +20: +20: +20: Loading extension module utils...Loading extension module utils...Loading extension module utils... +20: Loading extension module utils... +20: +20: +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: Loading extension module utils... +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +31: +31: Loading extension module utils...Loading extension module utils... +31: +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 8: + 8: Loading extension module utils...Loading extension module utils... + 8: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: +27: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +27: +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +27: +27: Loading extension module utils... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +18: +18: +18: Loading extension module utils...Loading extension module utils...Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: +18: +18: Loading extension module utils... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +26: +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +13: +13: Loading extension module utils...Loading extension module utils... +13: +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: Loading extension module utils... +13: +13: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 3: + 3: + 3: Loading extension module utils...Loading extension module utils...Loading extension module utils... + 3: + 3: + 3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 3: + 3: + 3: + 3: Loading extension module utils...Loading extension module utils...Loading extension module utils...Loading extension module utils... + 3: + 3: + 3: + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: +29: Loading extension module utils... +29: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +28: +28: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +28: +28: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +28: +28: +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +28: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +28: +28: +28: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +14: +14: Loading extension module utils...Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +14: +14: +14: +14: +14: Loading extension module utils...Loading extension module utils...Loading extension module utils... +14: +14: +14: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +14: +14: Loading extension module utils...Loading extension module utils... +14: +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: +16: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +22: +22: Loading extension module utils...Loading extension module utils... +22: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: +22: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: +22: +22: Loading extension module utils...Loading extension module utils... +22: +17: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +17: +17: +17: Loading extension module utils...Loading extension module utils...Loading extension module utils... +17: +17: +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +17: +17: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +16: +16: +16: Loading extension module utils...Loading extension module utils...Loading extension module utils... +16: +16: + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 7: + 7: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 7: Loading extension module utils... + 7: + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: + 7: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +16: +16: Loading extension module utils...Loading extension module utils... +16: +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +17: Loading extension module utils... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +17: +17: Loading extension module utils...Loading extension module utils... +17: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +16: +16: Loading extension module utils...Loading extension module utils... +16: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... + 9: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +25: +25: Loading extension module utils...Loading extension module utils... +25: +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +25: +25: Loading extension module utils... +25: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +10: +10: Loading extension module utils...Loading extension module utils... +10: +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +24: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +24: +24: Loading extension module utils...Loading extension module utils... +24: +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 2: + 2: Loading extension module utils...Loading extension module utils... + 2: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings + 0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/perplexity/3510897.out b/perplexity/3510897.out new file mode 100644 index 0000000000000000000000000000000000000000..a656e363aa45aadb7b7ad903c2cf20fb3b7f76f0 --- /dev/null +++ b/perplexity/3510897.out @@ -0,0 +1,42549 @@ +Model parameters: d_model 3072 ffw_size 12288 kv_size 128 n_heads 24 n_layers 36 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 2 --pipeline-model-parallel-size 1 --num-layers 36 --hidden-size 3072 --num-attention-heads 24 --kv-channels 128 --ffn-hidden-size 12288 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 2 --global-batch-size 1024 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-4b284bc4perplexityval --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --override-lr-scheduler --reset-progress --no-load-optim --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --eval-only true --tensorboard-dir tensorboard_4b284bc4perplexityval --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_4b284bc4perplexity --load checkpoints_4b284bc4perplexity --train-weighted-split-paths-path train1b5.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/3510897.json --zero-stage 0 +START 3510897: Sat 13 May 2023 09:34:16 AM EEST +12: +12: +12: ======================= ROCm System Management Interface ======================= +12: ================================= Concise Info ================================= +12: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +12: 0 47.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 2 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 4 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 6 40.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: ================================================================================ +12: ============================= End of ROCm SMI Log ============================== +11: +11: +11: ======================= ROCm System Management Interface ======================= +11: ================================= Concise Info ================================= +11: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +11: 0 49.0c 99.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 1 52.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 2 49.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 4 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 6 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: ================================================================================ +11: ============================= End of ROCm SMI Log ============================== + 9: + 9: + 9: ======================= ROCm System Management Interface ======================= + 9: ================================= Concise Info ================================= + 9: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 9: 0 50.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 2 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 4 47.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 6 45.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: ================================================================================ + 9: ============================= End of ROCm SMI Log ============================== + 0: + 0: + 0: ======================= ROCm System Management Interface ======================= + 0: ================================= Concise Info ================================= + 0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 0: 0 48.0c 101.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 2 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 4 48.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 6 42.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: ================================================================================ + 0: ============================= End of ROCm SMI Log ============================== +13: +13: +13: ======================= ROCm System Management Interface ======================= +13: ================================= Concise Info ================================= +13: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +13: 0 48.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 2 48.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 4 42.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 6 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 7 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: ================================================================================ +13: ============================= End of ROCm SMI Log ============================== +31: +31: +31: ======================= ROCm System Management Interface ======================= +31: ================================= Concise Info ================================= +31: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +31: 0 52.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 2 44.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 4 43.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 6 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: ================================================================================ +31: ============================= End of ROCm SMI Log ============================== +27: +27: +27: ======================= ROCm System Management Interface ======================= +27: ================================= Concise Info ================================= +27: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +27: 0 53.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 2 41.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 4 47.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 6 44.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: ================================================================================ +27: ============================= End of ROCm SMI Log ============================== +25: +25: +25: ======================= ROCm System Management Interface ======================= +25: ================================= Concise Info ================================= +25: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +25: 0 48.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 2 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 4 44.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 6 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: ================================================================================ +25: ============================= End of ROCm SMI Log ============================== + 2: + 2: + 2: ======================= ROCm System Management Interface ======================= + 2: ================================= Concise Info ================================= + 2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 2: 0 49.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 1 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 2 46.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 4 38.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 6 39.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: ================================================================================ + 2: ============================= End of ROCm SMI Log ============================== +17: +17: +17: ======================= ROCm System Management Interface ======================= +17: ================================= Concise Info ================================= +17: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +17: 0 49.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 2 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 3 38.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 4 45.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 6 43.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: ================================================================================ +17: ============================= End of ROCm SMI Log ============================== + 3: + 3: + 3: ======================= ROCm System Management Interface ======================= + 3: ================================= Concise Info ================================= + 3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 3: 0 46.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 2 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 4 50.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 6 44.0c 80.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 7 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: ================================================================================ + 3: ============================= End of ROCm SMI Log ============================== + 5: + 5: + 5: ======================= ROCm System Management Interface ======================= + 5: ================================= Concise Info ================================= + 5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 5: 0 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 2 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 4 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 6 36.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: ================================================================================ + 5: ============================= End of ROCm SMI Log ============================== +10: +10: +10: ======================= ROCm System Management Interface ======================= +10: ================================= Concise Info ================================= +10: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +10: 0 46.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 2 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 4 44.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 6 50.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: ================================================================================ +10: ============================= End of ROCm SMI Log ============================== +20: +20: +20: ======================= ROCm System Management Interface ======================= +20: ================================= Concise Info ================================= +20: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +20: 0 48.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 2 38.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 3 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 4 45.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 6 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: ================================================================================ +20: ============================= End of ROCm SMI Log ============================== +16: +16: +16: ======================= ROCm System Management Interface ======================= +16: ================================= Concise Info ================================= +16: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +16: 0 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 2 38.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 4 49.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 6 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: ================================================================================ +16: ============================= End of ROCm SMI Log ============================== +22: +22: +22: ======================= ROCm System Management Interface ======================= +22: ================================= Concise Info ================================= +22: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +22: 0 42.0c 101.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 2 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 4 38.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 6 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 7 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: ================================================================================ +22: ============================= End of ROCm SMI Log ============================== + 7: + 7: + 7: ======================= ROCm System Management Interface ======================= + 7: ================================= Concise Info ================================= + 7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 7: 0 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 2 41.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 3 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 4 42.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 6 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: ================================================================================ + 7: ============================= End of ROCm SMI Log ============================== + 4: + 4: + 4: ======================= ROCm System Management Interface ======================= + 4: ================================= Concise Info ================================= + 4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 4: 0 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 2 39.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 4 47.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 6 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: ================================================================================ + 4: ============================= End of ROCm SMI Log ============================== +26: +26: +26: ======================= ROCm System Management Interface ======================= +26: ================================= Concise Info ================================= +26: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +26: 0 45.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 2 48.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 4 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 6 40.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: ================================================================================ +26: ============================= End of ROCm SMI Log ============================== +14: +14: +14: ======================= ROCm System Management Interface ======================= +14: ================================= Concise Info ================================= +14: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +14: 0 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 2 45.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 4 46.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 6 35.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: ================================================================================ +14: ============================= End of ROCm SMI Log ============================== +24: +24: +24: ======================= ROCm System Management Interface ======================= +24: ================================= Concise Info ================================= +24: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +24: 0 46.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 2 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 4 47.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 6 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: ================================================================================ +24: ============================= End of ROCm SMI Log ============================== +19: +19: +19: ======================= ROCm System Management Interface ======================= +19: ================================= Concise Info ================================= +19: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +19: 0 45.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 2 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 3 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 4 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 6 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: ================================================================================ +19: ============================= End of ROCm SMI Log ============================== +23: +23: +23: ======================= ROCm System Management Interface ======================= +23: ================================= Concise Info ================================= +23: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +23: 0 47.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 2 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 4 48.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 6 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: ================================================================================ +23: ============================= End of ROCm SMI Log ============================== +28: +28: +28: ======================= ROCm System Management Interface ======================= +28: ================================= Concise Info ================================= +28: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +28: 0 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 1 53.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 2 37.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 3 37.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 4 46.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 6 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: ================================================================================ +28: ============================= End of ROCm SMI Log ============================== +30: +30: +30: ======================= ROCm System Management Interface ======================= +30: ================================= Concise Info ================================= +30: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +30: 0 51.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 2 39.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 4 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 6 41.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: ================================================================================ +30: ============================= End of ROCm SMI Log ============================== +21: +21: +21: ======================= ROCm System Management Interface ======================= +21: ================================= Concise Info ================================= +21: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +21: 0 49.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 1 52.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 2 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 4 42.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 6 39.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: ================================================================================ +21: ============================= End of ROCm SMI Log ============================== +29: +29: +29: ======================= ROCm System Management Interface ======================= +29: ================================= Concise Info ================================= +29: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +29: 0 47.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 2 46.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 4 48.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 6 45.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: ================================================================================ +29: ============================= End of ROCm SMI Log ============================== +15: +15: +15: ======================= ROCm System Management Interface ======================= +15: ================================= Concise Info ================================= +15: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +15: 0 46.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 2 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 4 46.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 6 39.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: ================================================================================ +15: ============================= End of ROCm SMI Log ============================== +18: +18: +18: ======================= ROCm System Management Interface ======================= +18: ================================= Concise Info ================================= +18: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +18: 0 42.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 2 39.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 4 43.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 5 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 6 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: ================================================================================ +18: ============================= End of ROCm SMI Log ============================== + 8: + 8: + 8: ======================= ROCm System Management Interface ======================= + 8: ================================= Concise Info ================================= + 8: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 8: 0 45.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 2 39.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 4 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 5 52.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 6 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: ================================================================================ + 8: ============================= End of ROCm SMI Log ============================== + 1: + 1: + 1: ======================= ROCm System Management Interface ======================= + 1: ================================= Concise Info ================================= + 1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 1: 0 44.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 2 39.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 4 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 6 40.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: ================================================================================ + 1: ============================= End of ROCm SMI Log ============================== + 6: + 6: + 6: ======================= ROCm System Management Interface ======================= + 6: ================================= Concise Info ================================= + 6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 6: 0 40.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 2 39.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 4 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 6 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: ================================================================================ + 6: ============================= End of ROCm SMI Log ============================== +12: Launching on nid006128 (12/32), master nid005327 port 9999, GPUs 8, CUDA: True +13: Launching on nid006129 (13/32), master nid005327 port 9999, GPUs 8, CUDA: True + 3: Launching on nid005330 (3/32), master nid005327 port 9999, GPUs 8, CUDA: True +11: Launching on nid006127 (11/32), master nid005327 port 9999, GPUs 8, CUDA: True +19: Launching on nid006135 (19/32), master nid005327 port 9999, GPUs 8, CUDA: True + 4: Launching on nid005331 (4/32), master nid005327 port 9999, GPUs 8, CUDA: True +26: Launching on nid006142 (26/32), master nid005327 port 9999, GPUs 8, CUDA: True +27: Launching on nid006143 (27/32), master nid005327 port 9999, GPUs 8, CUDA: True + 2: Launching on nid005329 (2/32), master nid005327 port 9999, GPUs 8, CUDA: True +25: Launching on nid006141 (25/32), master nid005327 port 9999, GPUs 8, CUDA: True +10: Launching on nid006126 (10/32), master nid005327 port 9999, GPUs 8, CUDA: True +24: Launching on nid006140 (24/32), master nid005327 port 9999, GPUs 8, CUDA: True +30: Launching on nid006146 (30/32), master nid005327 port 9999, GPUs 8, CUDA: True +16: Launching on nid006132 (16/32), master nid005327 port 9999, GPUs 8, CUDA: True +22: Launching on nid006138 (22/32), master nid005327 port 9999, GPUs 8, CUDA: True + 7: Launching on nid005334 (7/32), master nid005327 port 9999, GPUs 8, CUDA: True +23: Launching on nid006139 (23/32), master nid005327 port 9999, GPUs 8, CUDA: True + 9: Launching on nid006125 (9/32), master nid005327 port 9999, GPUs 8, CUDA: True +17: Launching on nid006133 (17/32), master nid005327 port 9999, GPUs 8, CUDA: True +31: Launching on nid006147 (31/32), master nid005327 port 9999, GPUs 8, CUDA: True + 5: Launching on nid005332 (5/32), master nid005327 port 9999, GPUs 8, CUDA: True +14: Launching on nid006130 (14/32), master nid005327 port 9999, GPUs 8, CUDA: True +29: Launching on nid006145 (29/32), master nid005327 port 9999, GPUs 8, CUDA: True +20: Launching on nid006136 (20/32), master nid005327 port 9999, GPUs 8, CUDA: True +15: Launching on nid006131 (15/32), master nid005327 port 9999, GPUs 8, CUDA: True +21: Launching on nid006137 (21/32), master nid005327 port 9999, GPUs 8, CUDA: True + 0: Launching on nid005327 (0/32), master nid005327 port 9999, GPUs 8, CUDA: True +28: Launching on nid006144 (28/32), master nid005327 port 9999, GPUs 8, CUDA: True + 1: Launching on nid005328 (1/32), master nid005327 port 9999, GPUs 8, CUDA: True + 8: Launching on nid005335 (8/32), master nid005327 port 9999, GPUs 8, CUDA: True + 6: Launching on nid005333 (6/32), master nid005327 port 9999, GPUs 8, CUDA: True +18: Launching on nid006134 (18/32), master nid005327 port 9999, GPUs 8, CUDA: True + 0: using world size: 256, data-parallel-size: 128, tensor-model-parallel size: 2, pipeline-model-parallel size: 1 + 0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. + 0: using torch.bfloat16 for parameters ... + 0: ------------------------ arguments ------------------------ + 0: abort_on_unmet_fused_kernel_constraints ......... False + 0: accumulate_allreduce_grads_in_fp32 .............. True + 0: adam_beta1 ...................................... 0.9 + 0: adam_beta2 ...................................... 0.999 + 0: adam_eps ........................................ 1e-08 + 0: adlr_autoresume ................................. False + 0: adlr_autoresume_interval ........................ 1000 + 0: apply_query_key_layer_scaling ................... True + 0: apply_residual_connection_post_layernorm ........ False + 0: attention_dropout ............................... 0.1 + 0: attention_softmax_in_fp32 ....................... False + 0: bert_binary_head ................................ True + 0: bert_load ....................................... None + 0: bf16 ............................................ True + 0: bias_dropout_fusion ............................. True + 0: bias_gelu_fusion ................................ True + 0: biencoder_projection_dim ........................ 0 + 0: biencoder_shared_query_context_model ............ False + 0: block_data_path ................................. None + 0: checkpoint_activations .......................... False + 0: checkpoint_in_cpu ............................... False + 0: checkpoint_num_layers ........................... 1 + 0: clip_grad ....................................... 1.0 + 0: codecarbon_dir .................................. None + 0: consumed_train_samples .......................... 0 + 0: consumed_train_tokens ........................... 0 + 0: consumed_valid_samples .......................... 0 + 0: contigious_checkpointing ........................ False + 0: cpu_optimizer ................................... False + 0: cpu_torch_adam .................................. False + 0: curriculum_learning ............................. False + 0: data_impl ....................................... mmap + 0: data_parallel_size .............................. 128 + 0: data_path ....................................... None + 0: dataloader_type ................................. single + 0: DDP_impl ........................................ local + 0: decoder_seq_length .............................. None + 0: deepscale ....................................... False + 0: deepscale_config ................................ None + 0: deepspeed ....................................... True + 0: deepspeed_activation_checkpointing .............. False + 0: deepspeed_config ................................ ds_configs/3510897.json + 0: deepspeed_mpi ................................... False + 0: distribute_checkpointed_activations ............. False + 0: distributed_backend ............................. nccl + 0: embed_layernorm ................................. False + 0: embedding_path .................................. None + 0: encoder_seq_length .............................. 2048 + 0: eod_mask_loss ................................... False + 0: eval_interval ................................... 1 + 0: eval_iters ...................................... 100 + 0: eval_only ....................................... True + 0: evidence_data_path .............................. None + 0: exit_duration_in_mins ........................... None + 0: exit_interval ................................... None + 0: ffn_hidden_size ................................. 12288 + 0: finetune ........................................ False + 0: fp16 ............................................ False + 0: fp16_lm_cross_entropy ........................... False + 0: fp32_residual_connection ........................ False + 0: gigaflos_no_embeds .............................. 0 + 0: global_batch_size ............................... 1024 + 0: glu_activation .................................. None + 0: hidden_dropout .................................. 0.1 + 0: hidden_size ..................................... 3072 + 0: hysteresis ...................................... 2 + 0: ict_head_size ................................... None + 0: ict_load ........................................ None + 0: img_dim ......................................... 224 + 0: indexer_batch_size .............................. 128 + 0: indexer_log_interval ............................ 1000 + 0: inference ....................................... False + 0: init_method_std ................................. 0.02 + 0: init_method_xavier_uniform ...................... False + 0: initial_loss_scale .............................. 4294967296 + 0: kill_switch_path ................................ kill-switch-4b284bc4perplexityval + 0: kv_channels ..................................... 128 + 0: layer_norm_fusion ............................... True + 0: layernorm_epsilon ............................... 1e-05 + 0: lazy_mpu_init ................................... None + 0: load ............................................ checkpoints_4b284bc4perplexity + 0: local_rank ...................................... None + 0: log_batch_size_to_tensorboard ................... True + 0: log_interval .................................... 10 + 0: log_learning_rate_to_tensorboard ................ True + 0: log_level ....................................... None + 0: log_level_replica ............................... None + 0: log_loss_scale_to_tensorboard ................... True + 0: log_num_zeros_in_grad ........................... False + 0: log_params_norm ................................. False + 0: log_path ........................................ None + 0: log_timers_to_tensorboard ....................... True + 0: log_validation_ppl_to_tensorboard ............... True + 0: loss_on_targets_only ............................ False + 0: loss_scale ...................................... None + 0: loss_scale_window ............................... 1000 + 0: lr .............................................. 0.0002 + 0: lr_decay_iters .................................. None + 0: lr_decay_samples ................................ 1 + 0: lr_decay_style .................................. cosine + 0: lr_decay_tokens ................................. None + 0: lr_warmup_fraction .............................. None + 0: lr_warmup_iters ................................. 0 + 0: lr_warmup_samples ............................... 0 + 0: make_vocab_size_divisible_by .................... 128 + 0: mask_prob ....................................... 0.15 + 0: masked_softmax_fusion ........................... True + 0: max_position_embeddings ......................... 2048 + 0: mean_noise_span_length .......................... None + 0: memory_centric_tiled_linear ..................... False + 0: merge_file ...................................... gpt2/merges.txt + 0: micro_batch_size ................................ 2 + 0: min_loss_scale .................................. 1.0 + 0: min_lr .......................................... 2e-05 + 0: mmap_warmup ..................................... False + 0: no_load_optim ................................... True + 0: no_load_rng ..................................... None + 0: no_save_optim ................................... None + 0: no_save_rng ..................................... None + 0: noise_density ................................... None + 0: num_attention_heads ............................. 24 + 0: num_channels .................................... 3 + 0: num_classes ..................................... 1000 + 0: num_layers ...................................... 36 + 0: num_layers_per_virtual_pipeline_stage ........... None + 0: num_workers ..................................... 2 + 0: onnx_safe ....................................... None + 0: openai_gelu ..................................... False + 0: optimizer ....................................... adam + 0: optimizer_fusion ................................ True + 0: override_lr_scheduler ........................... True + 0: pad_vocab_size_to ............................... None + 0: params_dtype .................................... torch.bfloat16 + 0: partition_activations ........................... False + 0: patch_dim ....................................... 16 + 0: pipeline_model_parallel_size .................... 1 + 0: position_embedding_type ......................... PositionEmbeddingType.absolute + 0: pp_partition_method ............................. None + 0: profile_backward ................................ False + 0: query_in_block_prob ............................. 0.1 + 0: rampup_batch_size ............................... None + 0: rank ............................................ 0 + 0: remote_device ................................... none + 0: reset_attention_mask ............................ False + 0: reset_position_ids .............................. False + 0: reset_progress .................................. True + 0: retriever_report_topk_accuracies ................ [] + 0: retriever_score_scaling ......................... False + 0: retriever_seq_length ............................ 256 + 0: reweight_loss_based_on_position_frequency ....... False + 0: sample_rate ..................................... 1.0 + 0: save ............................................ checkpoints_4b284bc4perplexity + 0: save_interval ................................... 1000 + 0: scatter_gather_tensors_in_pipeline .............. True + 0: scattered_embeddings ............................ False + 0: seed ............................................ 1234 + 0: seq_length ...................................... 2048 + 0: sgd_momentum .................................... 0.9 + 0: short_seq_prob .................................. 0.1 + 0: skip_train_iteration_range ...................... None + 0: split ........................................... None + 0: split_transformers .............................. False + 0: sync_tp_duplicated_parameters ................... False + 0: synchronize_each_layer .......................... False + 0: tensor_model_parallel_size ...................... 2 + 0: tensorboard_dir ................................. tensorboard_4b284bc4perplexityval + 0: tensorboard_log_interval ........................ 1 + 0: tensorboard_queue_size .......................... 5 + 0: test_weighted_split_paths ....................... None + 0: test_weighted_split_paths_path .................. None + 0: tile_factor ..................................... 1 + 0: titles_data_path ................................ None + 0: tokenizer_name_or_path .......................... None + 0: tokenizer_type .................................. GPT2BPETokenizer + 0: train_iters ..................................... None + 0: train_samples ................................... 1 + 0: train_tokens .................................... None + 0: train_weighted_split_names ...................... ['train'] + 0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document']] + 0: train_weighted_split_paths_path ................. None + 0: train_weighted_split_splits ..................... [['0:1']] + 0: train_weighted_split_weights .................... [['1.0']] + 0: universal_checkpoint ............................ False + 0: use_bnb_optimizer ............................... False + 0: use_checkpoint_lr_scheduler ..................... False + 0: use_contiguous_buffers_in_ddp ................... True + 0: use_cpu_initialization .......................... None + 0: use_one_sent_docs ............................... False + 0: use_pin_memory .................................. False + 0: valid_num_workers ............................... 2 + 0: valid_weighted_split_names ...................... ['validation'] + 0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] + 0: valid_weighted_split_paths_path ................. None + 0: valid_weighted_split_splits ..................... [['0:1']] + 0: valid_weighted_split_weights .................... [['1.0']] + 0: virtual_pipeline_model_parallel_size ............ None + 0: vocab_extra_ids ................................. 0 + 0: vocab_file ...................................... gpt2/vocab.json + 0: weight_decay .................................... 0.1 + 0: world_size ...................................... 256 + 0: zero_allgather_bucket_size ...................... 0.0 + 0: zero_contigious_gradients ....................... False + 0: zero_reduce_bucket_size ......................... 0.0 + 0: zero_reduce_scatter ............................. False + 0: zero_stage ...................................... 0 + 0: -------------------- end of arguments --------------------- + 0: setting number of micro-batches to constant 4 + 0: > building GPT2BPETokenizer tokenizer ... + 0: > padded vocab (size: 50257) with 175 dummy tokens (new size: 50432) + 0: DeepSpeed general environment info: + 0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] + 0: torch version .................... 1.13.0+rocm5.2 + 0: torch cuda version ............... None + 0: torch hip version ................ 5.2.21151-afdc89f8 + 0: nvcc version ..................... None + 0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] + 0: deepspeed info ................... 0.7.5, unknown, unknown + 0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 + 0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** + 0: > initializing torch distributed ... + 0: [2023-05-13 09:37:31,500] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +31: > setting tensorboard ... + 0: > initializing tensor model parallel with size 2 + 0: > initializing pipeline model parallel with size 1 + 0: > setting random seeds to 1234 ... + 0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 + 0: > compiling dataset index builder ... + 0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: make: Nothing to be done for 'default'. + 0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: >>> done with dataset index builder. Compilation time: 0.162 seconds + 0: > compiling and loading fused kernels ... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 87 + 0: [1/1] c++ scaled_upper_triang_masked_softmax_hip.cuda.o scaled_upper_triang_masked_softmax_hip.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/opt/rocm/lib -lamdhip64 -o scaled_upper_triang_masked_softmax_cuda.so + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 63 + 0: ninja: no work to do. + 0: >>> done with compiling and loading fused kernels. Compilation time: 32.356 seconds + 0: time to initialize megatron (seconds): 84.307 + 0: [after megatron is initialized] datetime: 2023-05-13 09:38:12 + 0: building GPT model ... + 0: [2023-05-13 09:38:12,470] [INFO] [utils.py:827:see_memory_usage] Before Building Model + 0: [2023-05-13 09:38:12,471] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB + 0: [2023-05-13 09:38:12,471] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.62 GB, percent = 8.1% + 0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None + 0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=0, model=1): 1, ProcessCoord(pipe=0, data=1, model=0): 2, ProcessCoord(pipe=0, data=1, model=1): 3, ProcessCoord(pipe=0, data=2, model=0): 4, ProcessCoord(pipe=0, data=2, model=1): 5, ProcessCoord(pipe=0, data=3, model=0): 6, ProcessCoord(pipe=0, data=3, model=1): 7, ProcessCoord(pipe=0, data=4, model=0): 8, ProcessCoord(pipe=0, data=4, model=1): 9, ProcessCoord(pipe=0, data=5, model=0): 10, ProcessCoord(pipe=0, data=5, model=1): 11, ProcessCoord(pipe=0, data=6, model=0): 12, ProcessCoord(pipe=0, data=6, model=1): 13, ProcessCoord(pipe=0, data=7, model=0): 14, ProcessCoord(pipe=0, data=7, model=1): 15, ProcessCoord(pipe=0, data=8, model=0): 16, ProcessCoord(pipe=0, data=8, model=1): 17, ProcessCoord(pipe=0, data=9, model=0): 18, ProcessCoord(pipe=0, data=9, model=1): 19, ProcessCoord(pipe=0, data=10, model=0): 20, ProcessCoord(pipe=0, data=10, model=1): 21, ProcessCoord(pipe=0, data=11, model=0): 22, ProcessCoord(pipe=0, data + 0: =11, model=1): 23, ProcessCoord(pipe=0, data=12, model=0): 24, ProcessCoord(pipe=0, data=12, model=1): 25, ProcessCoord(pipe=0, data=13, model=0): 26, ProcessCoord(pipe=0, data=13, model=1): 27, ProcessCoord(pipe=0, data=14, model=0): 28, ProcessCoord(pipe=0, data=14, model=1): 29, ProcessCoord(pipe=0, data=15, model=0): 30, ProcessCoord(pipe=0, data=15, model=1): 31, ProcessCoord(pipe=0, data=16, model=0): 32, ProcessCoord(pipe=0, data=16, model=1): 33, ProcessCoord(pipe=0, data=17, model=0): 34, ProcessCoord(pipe=0, data=17, model=1): 35, ProcessCoord(pipe=0, data=18, model=0): 36, ProcessCoord(pipe=0, data=18, model=1): 37, ProcessCoord(pipe=0, data=19, model=0): 38, ProcessCoord(pipe=0, data=19, model=1): 39, ProcessCoord(pipe=0, data=20, model=0): 40, ProcessCoord(pipe=0, data=20, model=1): 41, ProcessCoord(pipe=0, data=21, model=0): 42, ProcessCoord(pipe=0, data=21, model=1): 43, ProcessCoord(pipe=0, data=22, model=0): 44, ProcessCoord(pipe=0, data=22, model=1): 45, ProcessCoord(pipe=0, data=23, model=0 + 0: ): 46, ProcessCoord(pipe=0, data=23, model=1): 47, ProcessCoord(pipe=0, data=24, model=0): 48, ProcessCoord(pipe=0, data=24, model=1): 49, ProcessCoord(pipe=0, data=25, model=0): 50, ProcessCoord(pipe=0, data=25, model=1): 51, ProcessCoord(pipe=0, data=26, model=0): 52, ProcessCoord(pipe=0, data=26, model=1): 53, ProcessCoord(pipe=0, data=27, model=0): 54, ProcessCoord(pipe=0, data=27, model=1): 55, ProcessCoord(pipe=0, data=28, model=0): 56, ProcessCoord(pipe=0, data=28, model=1): 57, ProcessCoord(pipe=0, data=29, model=0): 58, ProcessCoord(pipe=0, data=29, model=1): 59, ProcessCoord(pipe=0, data=30, model=0): 60, ProcessCoord(pipe=0, data=30, model=1): 61, ProcessCoord(pipe=0, data=31, model=0): 62, ProcessCoord(pipe=0, data=31, model=1): 63, ProcessCoord(pipe=0, data=32, model=0): 64, ProcessCoord(pipe=0, data=32, model=1): 65, ProcessCoord(pipe=0, data=33, model=0): 66, ProcessCoord(pipe=0, data=33, model=1): 67, ProcessCoord(pipe=0, data=34, model=0): 68, ProcessCoord(pipe=0, data=34, model=1): 69, Proce + 0: ssCoord(pipe=0, data=35, model=0): 70, ProcessCoord(pipe=0, data=35, model=1): 71, ProcessCoord(pipe=0, data=36, model=0): 72, ProcessCoord(pipe=0, data=36, model=1): 73, ProcessCoord(pipe=0, data=37, model=0): 74, ProcessCoord(pipe=0, data=37, model=1): 75, ProcessCoord(pipe=0, data=38, model=0): 76, ProcessCoord(pipe=0, data=38, model=1): 77, ProcessCoord(pipe=0, data=39, model=0): 78, ProcessCoord(pipe=0, data=39, model=1): 79, ProcessCoord(pipe=0, data=40, model=0): 80, ProcessCoord(pipe=0, data=40, model=1): 81, ProcessCoord(pipe=0, data=41, model=0): 82, ProcessCoord(pipe=0, data=41, model=1): 83, ProcessCoord(pipe=0, data=42, model=0): 84, ProcessCoord(pipe=0, data=42, model=1): 85, ProcessCoord(pipe=0, data=43, model=0): 86, ProcessCoord(pipe=0, data=43, model=1): 87, ProcessCoord(pipe=0, data=44, model=0): 88, ProcessCoord(pipe=0, data=44, model=1): 89, ProcessCoord(pipe=0, data=45, model=0): 90, ProcessCoord(pipe=0, data=45, model=1): 91, ProcessCoord(pipe=0, data=46, model=0): 92, ProcessCoord(pipe + 0: =0, data=46, model=1): 93, ProcessCoord(pipe=0, data=47, model=0): 94, ProcessCoord(pipe=0, data=47, model=1): 95, ProcessCoord(pipe=0, data=48, model=0): 96, ProcessCoord(pipe=0, data=48, model=1): 97, ProcessCoord(pipe=0, data=49, model=0): 98, ProcessCoord(pipe=0, data=49, model=1): 99, ProcessCoord(pipe=0, data=50, model=0): 100, ProcessCoord(pipe=0, data=50, model=1): 101, ProcessCoord(pipe=0, data=51, model=0): 102, ProcessCoord(pipe=0, data=51, model=1): 103, ProcessCoord(pipe=0, data=52, model=0): 104, ProcessCoord(pipe=0, data=52, model=1): 105, ProcessCoord(pipe=0, data=53, model=0): 106, ProcessCoord(pipe=0, data=53, model=1): 107, ProcessCoord(pipe=0, data=54, model=0): 108, ProcessCoord(pipe=0, data=54, model=1): 109, ProcessCoord(pipe=0, data=55, model=0): 110, ProcessCoord(pipe=0, data=55, model=1): 111, ProcessCoord(pipe=0, data=56, model=0): 112, ProcessCoord(pipe=0, data=56, model=1): 113, ProcessCoord(pipe=0, data=57, model=0): 114, ProcessCoord(pipe=0, data=57, model=1): 115, ProcessCoord( + 0: pipe=0, data=58, model=0): 116, ProcessCoord(pipe=0, data=58, model=1): 117, ProcessCoord(pipe=0, data=59, model=0): 118, ProcessCoord(pipe=0, data=59, model=1): 119, ProcessCoord(pipe=0, data=60, model=0): 120, ProcessCoord(pipe=0, data=60, model=1): 121, ProcessCoord(pipe=0, data=61, model=0): 122, ProcessCoord(pipe=0, data=61, model=1): 123, ProcessCoord(pipe=0, data=62, model=0): 124, ProcessCoord(pipe=0, data=62, model=1): 125, ProcessCoord(pipe=0, data=63, model=0): 126, ProcessCoord(pipe=0, data=63, model=1): 127, ProcessCoord(pipe=0, data=64, model=0): 128, ProcessCoord(pipe=0, data=64, model=1): 129, ProcessCoord(pipe=0, data=65, model=0): 130, ProcessCoord(pipe=0, data=65, model=1): 131, ProcessCoord(pipe=0, data=66, model=0): 132, ProcessCoord(pipe=0, data=66, model=1): 133, ProcessCoord(pipe=0, data=67, model=0): 134, ProcessCoord(pipe=0, data=67, model=1): 135, ProcessCoord(pipe=0, data=68, model=0): 136, ProcessCoord(pipe=0, data=68, model=1): 137, ProcessCoord(pipe=0, data=69, model=0): 138, Pr + 0: ocessCoord(pipe=0, data=69, model=1): 139, ProcessCoord(pipe=0, data=70, model=0): 140, ProcessCoord(pipe=0, data=70, model=1): 141, ProcessCoord(pipe=0, data=71, model=0): 142, ProcessCoord(pipe=0, data=71, model=1): 143, ProcessCoord(pipe=0, data=72, model=0): 144, ProcessCoord(pipe=0, data=72, model=1): 145, ProcessCoord(pipe=0, data=73, model=0): 146, ProcessCoord(pipe=0, data=73, model=1): 147, ProcessCoord(pipe=0, data=74, model=0): 148, ProcessCoord(pipe=0, data=74, model=1): 149, ProcessCoord(pipe=0, data=75, model=0): 150, ProcessCoord(pipe=0, data=75, model=1): 151, ProcessCoord(pipe=0, data=76, model=0): 152, ProcessCoord(pipe=0, data=76, model=1): 153, ProcessCoord(pipe=0, data=77, model=0): 154, ProcessCoord(pipe=0, data=77, model=1): 155, ProcessCoord(pipe=0, data=78, model=0): 156, ProcessCoord(pipe=0, data=78, model=1): 157, ProcessCoord(pipe=0, data=79, model=0): 158, ProcessCoord(pipe=0, data=79, model=1): 159, ProcessCoord(pipe=0, data=80, model=0): 160, ProcessCoord(pipe=0, data=80, model= + 0: 1): 161, ProcessCoord(pipe=0, data=81, model=0): 162, ProcessCoord(pipe=0, data=81, model=1): 163, ProcessCoord(pipe=0, data=82, model=0): 164, ProcessCoord(pipe=0, data=82, model=1): 165, ProcessCoord(pipe=0, data=83, model=0): 166, ProcessCoord(pipe=0, data=83, model=1): 167, ProcessCoord(pipe=0, data=84, model=0): 168, ProcessCoord(pipe=0, data=84, model=1): 169, ProcessCoord(pipe=0, data=85, model=0): 170, ProcessCoord(pipe=0, data=85, model=1): 171, ProcessCoord(pipe=0, data=86, model=0): 172, ProcessCoord(pipe=0, data=86, model=1): 173, ProcessCoord(pipe=0, data=87, model=0): 174, ProcessCoord(pipe=0, data=87, model=1): 175, ProcessCoord(pipe=0, data=88, model=0): 176, ProcessCoord(pipe=0, data=88, model=1): 177, ProcessCoord(pipe=0, data=89, model=0): 178, ProcessCoord(pipe=0, data=89, model=1): 179, ProcessCoord(pipe=0, data=90, model=0): 180, ProcessCoord(pipe=0, data=90, model=1): 181, ProcessCoord(pipe=0, data=91, model=0): 182, ProcessCoord(pipe=0, data=91, model=1): 183, ProcessCoord(pipe=0, data + 0: =92, model=0): 184, ProcessCoord(pipe=0, data=92, model=1): 185, ProcessCoord(pipe=0, data=93, model=0): 186, ProcessCoord(pipe=0, data=93, model=1): 187, ProcessCoord(pipe=0, data=94, model=0): 188, ProcessCoord(pipe=0, data=94, model=1): 189, ProcessCoord(pipe=0, data=95, model=0): 190, ProcessCoord(pipe=0, data=95, model=1): 191, ProcessCoord(pipe=0, data=96, model=0): 192, ProcessCoord(pipe=0, data=96, model=1): 193, ProcessCoord(pipe=0, data=97, model=0): 194, ProcessCoord(pipe=0, data=97, model=1): 195, ProcessCoord(pipe=0, data=98, model=0): 196, ProcessCoord(pipe=0, data=98, model=1): 197, ProcessCoord(pipe=0, data=99, model=0): 198, ProcessCoord(pipe=0, data=99, model=1): 199, ProcessCoord(pipe=0, data=100, model=0): 200, ProcessCoord(pipe=0, data=100, model=1): 201, ProcessCoord(pipe=0, data=101, model=0): 202, ProcessCoord(pipe=0, data=101, model=1): 203, ProcessCoord(pipe=0, data=102, model=0): 204, ProcessCoord(pipe=0, data=102, model=1): 205, ProcessCoord(pipe=0, data=103, model=0): 206, Process + 0: Coord(pipe=0, data=103, model=1): 207, ProcessCoord(pipe=0, data=104, model=0): 208, ProcessCoord(pipe=0, data=104, model=1): 209, ProcessCoord(pipe=0, data=105, model=0): 210, ProcessCoord(pipe=0, data=105, model=1): 211, ProcessCoord(pipe=0, data=106, model=0): 212, ProcessCoord(pipe=0, data=106, model=1): 213, ProcessCoord(pipe=0, data=107, model=0): 214, ProcessCoord(pipe=0, data=107, model=1): 215, ProcessCoord(pipe=0, data=108, model=0): 216, ProcessCoord(pipe=0, data=108, model=1): 217, ProcessCoord(pipe=0, data=109, model=0): 218, ProcessCoord(pipe=0, data=109, model=1): 219, ProcessCoord(pipe=0, data=110, model=0): 220, ProcessCoord(pipe=0, data=110, model=1): 221, ProcessCoord(pipe=0, data=111, model=0): 222, ProcessCoord(pipe=0, data=111, model=1): 223, ProcessCoord(pipe=0, data=112, model=0): 224, ProcessCoord(pipe=0, data=112, model=1): 225, ProcessCoord(pipe=0, data=113, model=0): 226, ProcessCoord(pipe=0, data=113, model=1): 227, ProcessCoord(pipe=0, data=114, model=0): 228, ProcessCoord(pipe=0 + 0: , data=114, model=1): 229, ProcessCoord(pipe=0, data=115, model=0): 230, ProcessCoord(pipe=0, data=115, model=1): 231, ProcessCoord(pipe=0, data=116, model=0): 232, ProcessCoord(pipe=0, data=116, model=1): 233, ProcessCoord(pipe=0, data=117, model=0): 234, ProcessCoord(pipe=0, data=117, model=1): 235, ProcessCoord(pipe=0, data=118, model=0): 236, ProcessCoord(pipe=0, data=118, model=1): 237, ProcessCoord(pipe=0, data=119, model=0): 238, ProcessCoord(pipe=0, data=119, model=1): 239, ProcessCoord(pipe=0, data=120, model=0): 240, ProcessCoord(pipe=0, data=120, model=1): 241, ProcessCoord(pipe=0, data=121, model=0): 242, ProcessCoord(pipe=0, data=121, model=1): 243, ProcessCoord(pipe=0, data=122, model=0): 244, ProcessCoord(pipe=0, data=122, model=1): 245, ProcessCoord(pipe=0, data=123, model=0): 246, ProcessCoord(pipe=0, data=123, model=1): 247, ProcessCoord(pipe=0, data=124, model=0): 248, ProcessCoord(pipe=0, data=124, model=1): 249, ProcessCoord(pipe=0, data=125, model=0): 250, ProcessCoord(pipe=0, data=125, + 0: model=1): 251, ProcessCoord(pipe=0, data=126, model=0): 252, ProcessCoord(pipe=0, data=126, model=1): 253, ProcessCoord(pipe=0, data=127, model=0): 254, ProcessCoord(pipe=0, data=127, model=1): 255} + 0: [2023-05-13 09:38:17,957] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer + 0: stage=0 layers=43 + 0: 0: _to_float16 + 0: 1: EmbeddingPipe + 0: 2: + 0: 3: ParallelTransformerLayerPipe + 0: 4: ParallelTransformerLayerPipe + 0: 5: ParallelTransformerLayerPipe + 0: 6: ParallelTransformerLayerPipe + 0: 7: ParallelTransformerLayerPipe + 0: 8: ParallelTransformerLayerPipe + 0: 9: ParallelTransformerLayerPipe + 0: 10: ParallelTransformerLayerPipe + 0: 11: ParallelTransformerLayerPipe + 0: 12: ParallelTransformerLayerPipe + 0: 13: ParallelTransformerLayerPipe + 0: 14: ParallelTransformerLayerPipe + 0: 15: ParallelTransformerLayerPipe + 0: 16: ParallelTransformerLayerPipe + 0: 17: ParallelTransformerLayerPipe + 0: 18: ParallelTransformerLayerPipe + 0: 19: ParallelTransformerLayerPipe + 0: 20: ParallelTransformerLayerPipe + 0: 21: ParallelTransformerLayerPipe + 0: 22: ParallelTransformerLayerPipe + 0: 23: ParallelTransformerLayerPipe + 0: 24: ParallelTransformerLayerPipe + 0: 25: ParallelTransformerLayerPipe + 0: 26: ParallelTransformerLayerPipe + 0: 27: ParallelTransformerLayerPipe + 0: 28: ParallelTransformerLayerPipe + 0: 29: ParallelTransformerLayerPipe + 0: 30: ParallelTransformerLayerPipe + 0: 31: ParallelTransformerLayerPipe + 0: 32: ParallelTransformerLayerPipe + 0: 33: ParallelTransformerLayerPipe + 0: 34: ParallelTransformerLayerPipe + 0: 35: ParallelTransformerLayerPipe + 0: 36: ParallelTransformerLayerPipe + 0: 37: ParallelTransformerLayerPipe + 0: 38: ParallelTransformerLayerPipe + 0: 39: undo + 0: 40: MixedFusedLayerNorm + 0: 41: EmbeddingPipe + 0: 42: float16_to_fp32 + 0: loss: CrossEntropy + 0: [2023-05-13 09:38:18,395] [INFO] [utils.py:827:see_memory_usage] After Building Model + 0: [2023-05-13 09:38:18,396] [INFO] [utils.py:828:see_memory_usage] MA 4.0 GB Max_MA 4.0 GB CA 4.04 GB Max_CA 4 GB + 0: [2023-05-13 09:38:18,396] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.79 GB, percent = 8.1% + 0: setting training iterations to 0 + 0: > learning rate decay style: cosine + 0: DeepSpeed is enabled. + 0: [2023-05-13 09:38:18,399] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown + 0: [2023-05-13 09:38:23,675] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False + 0: [2023-05-13 09:38:23,676] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer + 0: [2023-05-13 09:38:23,676] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer + 0: [2023-05-13 09:38:23,697] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam + 0: [2023-05-13 09:38:23,697] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer + 0: [2023-05-13 09:38:23,830] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer + 0: [2023-05-13 09:38:23,831] [INFO] [utils.py:828:see_memory_usage] MA 3.99 GB Max_MA 4.01 GB CA 4.05 GB Max_CA 4 GB + 0: [2023-05-13 09:38:23,831] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.54 GB, percent = 8.3% + 0: ninja: no work to do. + 0: Time to load utils op: 0.3385152816772461 seconds + 0: Time to load utils op: 0.0008618831634521484 seconds + 1: ninja: no work to do. + 1: Time to load utils op: 0.17343664169311523 seconds + 0: Time to load utils op: 0.20445895195007324 seconds + 1: Time to load utils op: 0.20270276069641113 seconds + 1: Time to load utils op: 0.20256972312927246 secondsTime to load utils op: 0.2027130126953125 seconds + 1: + 6: Time to load utils op: 0.2097940444946289 secondsTime to load utils op: 0.21029043197631836 seconds + 6: Time to load utils op: 0.20903825759887695 seconds + 6: + 0: Time to load utils op: 0.0004558563232421875 seconds + 6: Time to load utils op: 0.21017837524414062 seconds + 3: Time to load utils op: 0.2180936336517334 seconds + 3: Time to load utils op: 0.21830511093139648 seconds + 3: Time to load utils op: 0.21822285652160645 seconds + 3: Time to load utils op: 0.21830391883850098 seconds + 5: Time to load utils op: 0.21964359283447266 secondsTime to load utils op: 0.21932458877563477 secondsTime to load utils op: 0.21889805793762207 secondsTime to load utils op: 0.21977472305297852 seconds + 5: + 5: + 5: + 0: Time to load utils op: 0.2210674285888672 seconds + 0: Time to load utils op: 0.22111773490905762 secondsTime to load utils op: 0.2214984893798828 seconds + 0: + 2: Time to load utils op: 0.22071361541748047 secondsTime to load utils op: 0.2206261157989502 secondsTime to load utils op: 0.22034907341003418 seconds + 2: Time to load utils op: 0.22038602828979492 seconds + 2: + 2: +18: Time to load utils op: 0.21410751342773438 secondsTime to load utils op: 0.21355915069580078 secondsTime to load utils op: 0.2135305404663086 seconds +18: +18: +18: Time to load utils op: 0.21389031410217285 seconds + 4: Time to load utils op: 0.21982645988464355 secondsTime to load utils op: 0.21979641914367676 secondsTime to load utils op: 0.22025489807128906 seconds + 4: Time to load utils op: 0.21978497505187988 seconds + 4: + 4: +22: Time to load utils op: 0.21166372299194336 secondsTime to load utils op: 0.2120361328125 seconds +22: +22: Time to load utils op: 0.21178317070007324 secondsTime to load utils op: 0.2119274139404297 seconds +22: +23: Time to load utils op: 0.2117619514465332 secondsTime to load utils op: 0.2113051414489746 seconds +23: +23: Time to load utils op: 0.21273088455200195 seconds +23: Time to load utils op: 0.21242809295654297 seconds +20: Time to load utils op: 0.21504592895507812 secondsTime to load utils op: 0.21456503868103027 secondsTime to load utils op: 0.21496915817260742 seconds +20: +20: +20: Time to load utils op: 0.21576690673828125 seconds +15: Time to load utils op: 0.2193315029144287 seconds +15: Time to load utils op: 0.21868681907653809 secondsTime to load utils op: 0.21910858154296875 seconds +15: +15: Time to load utils op: 0.21944904327392578 seconds + 0: Time to load utils op: 0.703455924987793 seconds + 8: Time to load utils op: 0.22313427925109863 secondsTime to load utils op: 0.2237720489501953 secondsTime to load utils op: 0.2233130931854248 secondsTime to load utils op: 0.22362422943115234 seconds + 8: + 8: + 8: + 0: Time to load utils op: 0.8191783428192139 seconds + 0: Time to load utils op: 0.8195176124572754 seconds +12: Time to load utils op: 0.22138667106628418 secondsTime to load utils op: 0.2199568748474121 secondsTime to load utils op: 0.22183752059936523 seconds +12: +12: +12: Time to load utils op: 0.22077202796936035 seconds +13: Time to load utils op: 0.2206263542175293 secondsTime to load utils op: 0.22088265419006348 seconds +13: Time to load utils op: 0.22066378593444824 secondsTime to load utils op: 0.22146034240722656 seconds +13: +13: +16: Time to load utils op: 0.21616601943969727 seconds +16: Time to load utils op: 0.2160778045654297 secondsTime to load utils op: 0.21601033210754395 secondsTime to load utils op: 0.21575617790222168 seconds +16: +16: +10: Time to load utils op: 0.22082972526550293 secondsTime to load utils op: 0.22042584419250488 seconds +10: +10: Time to load utils op: 0.22095537185668945 seconds +10: Time to load utils op: 0.22046470642089844 seconds +21: Time to load utils op: 0.21523571014404297 secondsTime to load utils op: 0.21662497520446777 secondsTime to load utils op: 0.21617341041564941 seconds +21: +21: Time to load utils op: 0.21704769134521484 seconds +21: +24: Time to load utils op: 0.21425271034240723 secondsTime to load utils op: 0.2153615951538086 secondsTime to load utils op: 0.21426057815551758 secondsTime to load utils op: 0.21422696113586426 seconds +24: +24: +24: +19: Time to load utils op: 0.21901154518127441 secondsTime to load utils op: 0.2185204029083252 secondsTime to load utils op: 0.21842384338378906 seconds +19: +19: +19: Time to load utils op: 0.21808314323425293 seconds + 7: Time to load utils op: 0.22524023056030273 secondsTime to load utils op: 0.2247905731201172 seconds + 7: Time to load utils op: 0.2244100570678711 seconds + 7: + 7: Time to load utils op: 0.22496700286865234 seconds +30: Time to load utils op: 0.21378254890441895 secondsTime to load utils op: 0.21317625045776367 secondsTime to load utils op: 0.21296429634094238 seconds +30: Time to load utils op: 0.21317315101623535 seconds +30: +30: +26: Time to load utils op: 0.2139878273010254 secondsTime to load utils op: 0.21388459205627441 seconds +26: +26: Time to load utils op: 0.21388864517211914 seconds +26: Time to load utils op: 0.21385407447814941 seconds +17: Time to load utils op: 0.21987485885620117 seconds +17: Time to load utils op: 0.22022175788879395 seconds +17: Time to load utils op: 0.22039365768432617 secondsTime to load utils op: 0.21925902366638184 seconds +17: +14: Time to load utils op: 0.22120189666748047 secondsTime to load utils op: 0.2214972972869873 seconds +14: +14: Time to load utils op: 0.22138547897338867 secondsTime to load utils op: 0.22153782844543457 seconds +14: +25: Time to load utils op: 0.21500205993652344 secondsTime to load utils op: 0.21543264389038086 secondsTime to load utils op: 0.21505165100097656 seconds +25: +25: Time to load utils op: 0.2151637077331543 seconds +25: +28: Time to load utils op: 0.21405839920043945 secondsTime to load utils op: 0.21419572830200195 secondsTime to load utils op: 0.21392512321472168 seconds +28: +28: +28: Time to load utils op: 0.21405339241027832 seconds +27: Time to load utils op: 0.2157447338104248 secondsTime to load utils op: 0.2158825397491455 secondsTime to load utils op: 0.21530604362487793 secondsTime to load utils op: 0.21547937393188477 seconds +27: +27: +27: +31: Time to load utils op: 0.21260571479797363 secondsTime to load utils op: 0.21219348907470703 seconds +31: Time to load utils op: 0.21248602867126465 seconds +31: +31: Time to load utils op: 0.21365594863891602 seconds +29: Time to load utils op: 0.2135617733001709 secondsTime to load utils op: 0.2134408950805664 secondsTime to load utils op: 0.21346354484558105 seconds +29: +29: +29: Time to load utils op: 0.2136685848236084 seconds + 4: Time to load utils op: 0.8188607692718506 seconds + 4: Time to load utils op: 0.8204503059387207 seconds + 4: Time to load utils op: 0.8186764717102051 seconds + 4: Time to load utils op: 0.8199427127838135 seconds + 6: Time to load utils op: 0.8202061653137207 secondsTime to load utils op: 0.8204877376556396 seconds + 6: + 1: Time to load utils op: 0.8234553337097168 seconds + 6: Time to load utils op: 0.8215372562408447 seconds + 1: Time to load utils op: 0.8230545520782471 seconds + 1: Time to load utils op: 0.8234179019927979 seconds + 1: Time to load utils op: 0.8233435153961182 seconds + 6: Time to load utils op: 0.8207550048828125 seconds + 7: Time to load utils op: 0.820075273513794 seconds + 7: Time to load utils op: 0.819669246673584 seconds + 7: Time to load utils op: 0.8200478553771973 seconds + 7: Time to load utils op: 0.8198635578155518 seconds +15: Time to load utils op: 0.8174784183502197 seconds +15: Time to load utils op: 0.8169138431549072 seconds +15: Time to load utils op: 0.8173720836639404 seconds + 5: Time to load utils op: 0.8193981647491455 seconds +15: Time to load utils op: 0.8172154426574707 seconds + 5: Time to load utils op: 0.8195059299468994 seconds + 5: Time to load utils op: 0.819561243057251 seconds +23: Time to load utils op: 0.8131294250488281 seconds +23: Time to load utils op: 0.8131191730499268 seconds + 3: Time to load utils op: 0.8203678131103516 seconds + 8: Time to load utils op: 0.8210098743438721 secondsTime to load utils op: 0.8209042549133301 seconds + 8: +17: Time to load utils op: 0.8163199424743652 seconds + 8: Time to load utils op: 0.8209483623504639 seconds + 5: Time to load utils op: 0.8200552463531494 seconds +17: Time to load utils op: 0.8174681663513184 seconds +23: Time to load utils op: 0.8145051002502441 secondsTime to load utils op: 0.813298225402832 seconds +23: +17: Time to load utils op: 0.8166708946228027 seconds +18: Time to load utils op: 0.8129844665527344 seconds + 3: Time to load utils op: 0.820915699005127 seconds + 3: Time to load utils op: 0.8209171295166016 seconds + 3: Time to load utils op: 0.8210220336914062 seconds + 8: Time to load utils op: 0.8221969604492188 seconds +18: Time to load utils op: 0.8131382465362549 seconds +13: Time to load utils op: 0.8157505989074707 seconds +14: Time to load utils op: 0.8151829242706299 seconds +17: Time to load utils op: 0.8174448013305664 seconds +12: Time to load utils op: 0.818885087966919 seconds +18: Time to load utils op: 0.8133718967437744 seconds +31: Time to load utils op: 0.8100409507751465 seconds +14: Time to load utils op: 0.815349817276001 seconds +31: Time to load utils op: 0.8100597858428955 seconds +12: Time to load utils op: 0.8202826976776123 seconds +13: Time to load utils op: 0.8161559104919434 secondsTime to load utils op: 0.8161623477935791 seconds +13: +13: Time to load utils op: 0.816169023513794 seconds +14: Time to load utils op: 0.81549072265625 seconds +18: Time to load utils op: 0.8136107921600342 seconds + 2: Time to load utils op: 0.8215911388397217 seconds +16: Time to load utils op: 0.8139421939849854 seconds + 2: Time to load utils op: 0.821660041809082 seconds +14: Time to load utils op: 0.8155927658081055 seconds +12: Time to load utils op: 0.8209953308105469 seconds +30: Time to load utils op: 0.8094375133514404 seconds +19: Time to load utils op: 0.8161923885345459 seconds +20: Time to load utils op: 0.8163821697235107 seconds +16: Time to load utils op: 0.8140931129455566 seconds +30: Time to load utils op: 0.8095684051513672 seconds +31: Time to load utils op: 0.8112893104553223 seconds +22: Time to load utils op: 0.811058521270752 seconds +19: Time to load utils op: 0.8157331943511963 seconds +12: Time to load utils op: 0.820192813873291 seconds +30: Time to load utils op: 0.8097267150878906 seconds +16: Time to load utils op: 0.8141825199127197 seconds +31: Time to load utils op: 0.8100020885467529 seconds +21: Time to load utils op: 0.8161723613739014 seconds +21: Time to load utils op: 0.8153421878814697 seconds +20: Time to load utils op: 0.8166840076446533 seconds + 2: Time to load utils op: 0.8219728469848633 seconds +27: Time to load utils op: 0.8122568130493164 seconds +10: Time to load utils op: 0.8176171779632568 secondsTime to load utils op: 0.8176188468933105 seconds +10: +19: Time to load utils op: 0.8161313533782959 seconds +19: Time to load utils op: 0.8165450096130371 seconds +27: Time to load utils op: 0.8126468658447266 seconds +10: Time to load utils op: 0.8176629543304443 seconds +20: Time to load utils op: 0.8163323402404785 seconds +30: Time to load utils op: 0.8096628189086914 seconds +10: Time to load utils op: 0.8177781105041504 seconds +20: Time to load utils op: 0.8167648315429688 seconds +21: Time to load utils op: 0.8165178298950195 seconds +25: Time to load utils op: 0.8097057342529297 seconds +16: Time to load utils op: 0.8145802021026611 seconds +27: Time to load utils op: 0.813082218170166 seconds + 2: Time to load utils op: 0.8223485946655273 seconds +22: Time to load utils op: 0.811572790145874 secondsTime to load utils op: 0.8115696907043457 seconds +22: +22: Time to load utils op: 0.8115835189819336 seconds +27: Time to load utils op: 0.8121511936187744 seconds +26: Time to load utils op: 0.8091006278991699 seconds +26: Time to load utils op: 0.8091576099395752 seconds +25: Time to load utils op: 0.8099825382232666 seconds +21: Time to load utils op: 0.8168990612030029 seconds +25: Time to load utils op: 0.8100490570068359 seconds +24: Time to load utils op: 0.8106281757354736 secondsTime to load utils op: 0.8106176853179932 seconds +24: +24: Time to load utils op: 0.8106956481933594 seconds +26: Time to load utils op: 0.8093066215515137 seconds +25: Time to load utils op: 0.8102068901062012 seconds +24: Time to load utils op: 0.8108184337615967 seconds +26: Time to load utils op: 0.8094544410705566 seconds +28: Time to load utils op: 0.8089694976806641 seconds +28: Time to load utils op: 0.8091881275177002 seconds +28: Time to load utils op: 0.8092525005340576 seconds +28: Time to load utils op: 0.8093390464782715 seconds +29: Time to load utils op: 0.8085148334503174 seconds +29: Time to load utils op: 0.8085770606994629 seconds +29: Time to load utils op: 0.8086540699005127 seconds + 9: Time to load utils op: 0.2037675380706787 seconds + 9: Time to load utils op: 0.20977330207824707 seconds + 9: Time to load utils op: 0.209791898727417 seconds + 9: Time to load utils op: 0.20965099334716797 secondsTime to load utils op: 0.8215229511260986 seconds +29: Time to load utils op: 0.8088221549987793 seconds + 9: + 9: Time to load utils op: 0.821526288986206 seconds + 9: Time to load utils op: 0.8215420246124268 seconds + 9: Time to load utils op: 0.8215501308441162 seconds +11: Time to load utils op: 0.23752284049987793 secondsTime to load utils op: 0.23736119270324707 secondsTime to load utils op: 0.2383267879486084 seconds +11: +11: +11: Time to load utils op: 0.23796391487121582 seconds +11: Time to load utils op: 0.8301656246185303 seconds +11: Time to load utils op: 0.8303980827331543 seconds +11: Time to load utils op: 0.8302457332611084 seconds +11: Time to load utils op: 0.8297433853149414 seconds + 0: Time to load utils op: 0.0004220008850097656 seconds + 0: Time to load utils op: 0.0005252361297607422 seconds + 0: Time to load utils op: 0.0004112720489501953 seconds + 0: Time to load utils op: 0.0003955364227294922 seconds + 0: Time to load utils op: 0.0003914833068847656 seconds + 1: Time to load utils op: 0.0005743503570556641 seconds + 1: Time to load utils op: 0.0005664825439453125 secondsTime to load utils op: 0.0005867481231689453 seconds + 1: + 1: Time to load utils op: 0.0005867481231689453 seconds + 1: Time to load utils op: 0.0006017684936523438 seconds + 1: Time to load utils op: 0.0006480216979980469 seconds + 1: Time to load utils op: 0.0007236003875732422 seconds + 1: Time to load utils op: 0.0004937648773193359 seconds + 0: [2023-05-13 09:38:24,667] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 + 0: [2023-05-13 09:38:24,667] [INFO] [utils.py:828:see_memory_usage] MA 3.99 GB Max_MA 3.99 GB CA 4.05 GB Max_CA 4 GB + 0: [2023-05-13 09:38:24,667] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.71 GB, percent = 8.3% + 6: Time to load utils op: 0.0004973411560058594 seconds + 6: Time to load utils op: 0.0005459785461425781 seconds + 6: Time to load utils op: 0.0005164146423339844 seconds + 6: Time to load utils op: 0.0005617141723632812 seconds + 6: Time to load utils op: 0.0004839897155761719 secondsTime to load utils op: 0.0004894733428955078 secondsTime to load utils op: 0.0004875659942626953 seconds + 6: + 6: + 6: Time to load utils op: 0.0004162788391113281 seconds +19: Time to load utils op: 0.0005207061767578125 seconds +19: Time to load utils op: 0.0005249977111816406 seconds +19: Time to load utils op: 0.00045990943908691406 secondsTime to load utils op: 0.0004277229309082031 seconds +19: +19: Time to load utils op: 0.0004253387451171875 seconds +19: Time to load utils op: 0.0005426406860351562 seconds +19: Time to load utils op: 0.00054168701171875 seconds +19: Time to load utils op: 0.00043463706970214844 seconds +12: Time to load utils op: 0.0005590915679931641 seconds +12: Time to load utils op: 0.0004456043243408203 seconds +12: Time to load utils op: 0.0005626678466796875 seconds +12: Time to load utils op: 0.0005893707275390625 seconds +12: Time to load utils op: 0.0005779266357421875 seconds +12: Time to load utils op: 0.000576019287109375 seconds +12: Time to load utils op: 0.0005729198455810547 seconds +12: Time to load utils op: 0.0006308555603027344 seconds +15: Time to load utils op: 0.0004787445068359375 seconds +15: Time to load utils op: 0.0004138946533203125 seconds +15: Time to load utils op: 0.0004253387451171875 seconds +15: Time to load utils op: 0.0005733966827392578 seconds +15: Time to load utils op: 0.0005872249603271484 seconds +15: Time to load utils op: 0.0008981227874755859 seconds +15: Time to load utils op: 0.000888824462890625 secondsTime to load utils op: 0.0008649826049804688 seconds +15: + 4: Time to load utils op: 0.0006787776947021484 seconds +11: Time to load utils op: 0.0005409717559814453 seconds +11: Time to load utils op: 0.0005567073822021484 seconds +11: Time to load utils op: 0.0004253387451171875 seconds +11: Time to load utils op: 0.0004425048828125 seconds + 4: Time to load utils op: 0.0009665489196777344 seconds +11: Time to load utils op: 0.0005469322204589844 seconds +11: Time to load utils op: 0.0006089210510253906 seconds +11: Time to load utils op: 0.0005769729614257812 seconds +11: Time to load utils op: 0.0005919933319091797 seconds + 4: Time to load utils op: 0.0013887882232666016 seconds + 4: Time to load utils op: 0.0012679100036621094 seconds + 4: Time to load utils op: 0.0013675689697265625 seconds + 4: Time to load utils op: 0.0013744831085205078 seconds + 4: Time to load utils op: 0.0014505386352539062 seconds + 4: Time to load utils op: 0.0014569759368896484 seconds +21: Time to load utils op: 0.0005168914794921875 seconds +21: Time to load utils op: 0.000545501708984375 seconds +21: Time to load utils op: 0.0005445480346679688 secondsTime to load utils op: 0.0006089210510253906 seconds +21: +21: Time to load utils op: 0.0006146430969238281 seconds +21: Time to load utils op: 0.0005593299865722656 seconds +21: Time to load utils op: 0.0006177425384521484 seconds +21: Time to load utils op: 0.0006155967712402344 seconds +23: Time to load utils op: 0.0004703998565673828 seconds +23: Time to load utils op: 0.0005517005920410156 seconds +23: Time to load utils op: 0.00045418739318847656 seconds +23: Time to load utils op: 0.00047469139099121094 seconds +23: Time to load utils op: 0.0004963874816894531 seconds +23: Time to load utils op: 0.0005383491516113281 seconds +23: Time to load utils op: 0.0005562305450439453 seconds +23: Time to load utils op: 0.0005955696105957031 seconds +20: Time to load utils op: 0.0005481243133544922 seconds +20: Time to load utils op: 0.0004127025604248047 seconds +20: Time to load utils op: 0.00045609474182128906 seconds +20: Time to load utils op: 0.0004792213439941406 secondsTime to load utils op: 0.00047969818115234375 seconds +20: +20: Time to load utils op: 0.0005197525024414062 seconds +20: Time to load utils op: 0.0005407333374023438 seconds +20: Time to load utils op: 0.0006089210510253906 seconds + 0: [2023-05-13 09:38:24,791] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 +31: Time to load utils op: 0.0005588531494140625 seconds +31: Time to load utils op: 0.0004303455352783203 seconds +31: Time to load utils op: 0.0004024505615234375 seconds +31: Time to load utils op: 0.0006949901580810547 seconds +31: Time to load utils op: 0.0008933544158935547 seconds +31: Time to load utils op: 0.0008282661437988281 seconds + 0: [2023-05-13 09:38:24,792] [INFO] [utils.py:828:see_memory_usage] MA 8.04 GB Max_MA 8.04 GB CA 10.11 GB Max_CA 10 GB +31: Time to load utils op: 0.0010352134704589844 secondsTime to load utils op: 0.0011200904846191406 seconds +31: + 0: [2023-05-13 09:38:24,792] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.71 GB, percent = 8.3% + 8: Time to load utils op: 0.0005242824554443359 seconds + 8: Time to load utils op: 0.00042939186096191406 seconds + 8: Time to load utils op: 0.0005445480346679688 seconds + 8: Time to load utils op: 0.0005488395690917969 seconds + 8: Time to load utils op: 0.0009348392486572266 seconds + 8: Time to load utils op: 0.0010166168212890625 seconds + 8: Time to load utils op: 0.000993967056274414 secondsTime to load utils op: 0.0009758472442626953 seconds + 8: +27: Time to load utils op: 0.0005214214324951172 seconds +27: Time to load utils op: 0.0005283355712890625 secondsTime to load utils op: 0.00044608116149902344 seconds +27: +27: Time to load utils op: 0.00039315223693847656 seconds +18: Time to load utils op: 0.0005781650543212891 seconds +18: Time to load utils op: 0.0005419254302978516 seconds +18: Time to load utils op: 0.0005719661712646484 seconds +27: Time to load utils op: 0.00060272216796875 secondsTime to load utils op: 0.0005869865417480469 secondsTime to load utils op: 0.0006010532379150391 seconds +27: +27: +27: Time to load utils op: 0.0006163120269775391 seconds +18: Time to load utils op: 0.0006213188171386719 seconds +18: Time to load utils op: 0.0008151531219482422 secondsTime to load utils op: 0.0008497238159179688 seconds +18: +18: Time to load utils op: 0.0007853507995605469 seconds +18: Time to load utils op: 0.0008544921875 seconds +13: Time to load utils op: 0.0005779266357421875 seconds +13: Time to load utils op: 0.0008420944213867188 seconds +26: Time to load utils op: 0.0008213520050048828 seconds +26: Time to load utils op: 0.0009386539459228516 seconds +26: Time to load utils op: 0.0009365081787109375 seconds +26: Time to load utils op: 0.0009109973907470703 seconds +26: Time to load utils op: 0.0009062290191650391 secondsTime to load utils op: 0.0008313655853271484 seconds +26: +26: Time to load utils op: 0.0008742809295654297 seconds +13: Time to load utils op: 0.0009415149688720703 secondsTime to load utils op: 0.0009164810180664062 seconds +13: +13: Time to load utils op: 0.0008716583251953125 secondsTime to load utils op: 0.0008990764617919922 seconds +13: +26: Time to load utils op: 0.0009722709655761719 seconds +13: Time to load utils op: 0.0008845329284667969 seconds +13: Time to load utils op: 0.0009665489196777344 seconds + 3: Time to load utils op: 0.0006110668182373047 secondsTime to load utils op: 0.0004947185516357422 secondsTime to load utils op: 0.0005159378051757812 secondsTime to load utils op: 0.0005288124084472656 secondsTime to load utils op: 0.0005738735198974609 seconds + 3: + 3: + 3: + 3: Time to load utils op: 0.000591278076171875 seconds + 3: + 3: Time to load utils op: 0.0005655288696289062 seconds + 3: Time to load utils op: 0.0005426406860351562 seconds + 0: [2023-05-13 09:38:24,900] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 + 0: [2023-05-13 09:38:24,900] [INFO] [utils.py:828:see_memory_usage] MA 8.04 GB Max_MA 8.04 GB CA 10.11 GB Max_CA 10 GB + 0: [2023-05-13 09:38:24,901] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.71 GB, percent = 8.3% + 5: Time to load utils op: 0.0006794929504394531 seconds +29: Time to load utils op: 0.0005888938903808594 secondsTime to load utils op: 0.0005929470062255859 secondsTime to load utils op: 0.0005469322204589844 seconds +29: +29: +29: Time to load utils op: 0.0005071163177490234 seconds + 5: Time to load utils op: 0.0008115768432617188 seconds + 5: Time to load utils op: 0.0007660388946533203 seconds + 5: Time to load utils op: 0.0006656646728515625 seconds + 5: Time to load utils op: 0.0008208751678466797 seconds +28: Time to load utils op: 0.0008311271667480469 seconds +28: Time to load utils op: 0.0007429122924804688 secondsTime to load utils op: 0.0007848739624023438 seconds +28: +28: Time to load utils op: 0.000812530517578125 secondsTime to load utils op: 0.0008285045623779297 seconds +28: Time to load utils op: 0.0007367134094238281 seconds +28: +28: Time to load utils op: 0.0007886886596679688 seconds +29: Time to load utils op: 0.0008172988891601562 seconds +28: Time to load utils op: 0.0008504390716552734 seconds + 5: Time to load utils op: 0.0008101463317871094 secondsTime to load utils op: 0.0007555484771728516 seconds + 5: +29: Time to load utils op: 0.0008301734924316406 seconds + 5: Time to load utils op: 0.0007801055908203125 seconds +29: Time to load utils op: 0.0008714199066162109 secondsTime to load utils op: 0.0008349418640136719 seconds +29: +14: Time to load utils op: 0.0008070468902587891 secondsTime to load utils op: 0.0008563995361328125 seconds +14: +14: Time to load utils op: 0.000812530517578125 secondsTime to load utils op: 0.0007684230804443359 seconds +14: Time to load utils op: 0.0008273124694824219 seconds +14: +14: Time to load utils op: 0.0008766651153564453 seconds +14: Time to load utils op: 0.0007996559143066406 seconds +14: Time to load utils op: 0.0009453296661376953 seconds + 0: [2023-05-13 09:38:25,008] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 + 0: [2023-05-13 09:38:25,009] [INFO] [utils.py:828:see_memory_usage] MA 11.92 GB Max_MA 11.92 GB CA 15.91 GB Max_CA 16 GB + 0: [2023-05-13 09:38:25,009] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.71 GB, percent = 8.3% +22: Time to load utils op: 0.0006818771362304688 seconds +22: Time to load utils op: 0.0007138252258300781 seconds +22: Time to load utils op: 0.000720977783203125 seconds + 7: Time to load utils op: 0.0007319450378417969 seconds +22: Time to load utils op: 0.0008525848388671875 secondsTime to load utils op: 0.0008311271667480469 secondsTime to load utils op: 0.0009055137634277344 secondsTime to load utils op: 0.0008730888366699219 seconds +22: +22: +22: +22: Time to load utils op: 0.0008389949798583984 seconds +17: Time to load utils op: 0.0006401538848876953 secondsTime to load utils op: 0.0007300376892089844 secondsTime to load utils op: 0.0008528232574462891 seconds +17: +17: +16: Time to load utils op: 0.0007684230804443359 secondsTime to load utils op: 0.0006892681121826172 seconds +16: +16: Time to load utils op: 0.0007290840148925781 seconds +17: Time to load utils op: 0.0007760524749755859 seconds +17: Time to load utils op: 0.0007860660552978516 seconds + 7: Time to load utils op: 0.0008890628814697266 seconds + 9: Time to load utils op: 0.0007548332214355469 seconds +17: Time to load utils op: 0.0009047985076904297 seconds + 7: Time to load utils op: 0.0008666515350341797 seconds + 7: Time to load utils op: 0.0009248256683349609 seconds +16: Time to load utils op: 0.0008723735809326172 seconds +16: Time to load utils op: 0.0009655952453613281 seconds + 7: Time to load utils op: 0.0008957386016845703 seconds + 7: Time to load utils op: 0.0008530616760253906 seconds +16: Time to load utils op: 0.0008425712585449219 seconds +17: Time to load utils op: 0.0008237361907958984 seconds + 7: Time to load utils op: 0.0008604526519775391 seconds +17: Time to load utils op: 0.0009064674377441406 seconds +16: Time to load utils op: 0.0010700225830078125 seconds +16: Time to load utils op: 0.0010771751403808594 seconds + 7: Time to load utils op: 0.0009980201721191406 seconds +30: Time to load utils op: 0.0005764961242675781 seconds +30: Time to load utils op: 0.0005540847778320312 seconds + 9: Time to load utils op: 0.0005059242248535156 seconds + 9: Time to load utils op: 0.0004925727844238281 seconds + 9: Time to load utils op: 0.0004956722259521484 seconds + 9: Time to load utils op: 0.0005011558532714844 seconds +30: Time to load utils op: 0.0007419586181640625 seconds + 9: Time to load utils op: 0.0004591941833496094 seconds +30: Time to load utils op: 0.001041412353515625 seconds +30: Time to load utils op: 0.0008480548858642578 seconds + 9: Time to load utils op: 0.0004911422729492188 seconds + 9: Time to load utils op: 0.0004885196685791016 seconds +30: Time to load utils op: 0.0009348392486572266 secondsTime to load utils op: 0.0010924339294433594 seconds +30: +30: Time to load utils op: 0.0009162425994873047 seconds +25: Time to load utils op: 0.0007801055908203125 seconds +10: Time to load utils op: 0.0007908344268798828 seconds +25: Time to load utils op: 0.0007960796356201172 secondsTime to load utils op: 0.0008051395416259766 seconds +25: +10: Time to load utils op: 0.0007925033569335938 seconds +24: Time to load utils op: 0.0006501674652099609 seconds +25: Time to load utils op: 0.0008268356323242188 secondsTime to load utils op: 0.0007724761962890625 seconds +25: +25: Time to load utils op: 0.0008387565612792969 secondsTime to load utils op: 0.0008137226104736328 seconds +25: +10: Time to load utils op: 0.0008623600006103516 seconds +10: Time to load utils op: 0.0008130073547363281 seconds +10: Time to load utils op: 0.0008103847503662109 seconds +24: Time to load utils op: 0.0008149147033691406 secondsTime to load utils op: 0.0008473396301269531 seconds +24: +10: Time to load utils op: 0.0008878707885742188 seconds +10: Time to load utils op: 0.0010578632354736328 seconds +25: Time to load utils op: 0.0012350082397460938 seconds +10: Time to load utils op: 0.001065969467163086 seconds +24: Time to load utils op: 0.0009250640869140625 secondsTime to load utils op: 0.0009021759033203125 seconds +24: +24: Time to load utils op: 0.0009844303131103516 secondsTime to load utils op: 0.0009586811065673828 seconds +24: +24: Time to load utils op: 0.0009605884552001953 seconds + 2: Time to load utils op: 0.0013964176177978516 seconds + 2: Time to load utils op: 0.0013592243194580078 seconds + 2: Time to load utils op: 0.00170135498046875 seconds + 2: Time to load utils op: 0.002004384994506836 seconds + 2: Time to load utils op: 0.0018236637115478516 seconds + 2: Time to load utils op: 0.0019223690032958984 seconds + 2: Time to load utils op: 0.002058267593383789 seconds + 2: Time to load utils op: 0.0019865036010742188 seconds + 0: [2023-05-13 09:38:25,114] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 + 0: [2023-05-13 09:38:25,115] [INFO] [utils.py:828:see_memory_usage] MA 11.92 GB Max_MA 11.92 GB CA 15.91 GB Max_CA 16 GB + 0: [2023-05-13 09:38:25,115] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.71 GB, percent = 8.3% + 0: [2023-05-13 09:38:25,223] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 + 0: [2023-05-13 09:38:25,224] [INFO] [utils.py:828:see_memory_usage] MA 11.93 GB Max_MA 11.93 GB CA 15.91 GB Max_CA 16 GB + 0: [2023-05-13 09:38:25,224] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.71 GB, percent = 8.3% + 0: [2023-05-13 09:38:25,326] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer + 0: [2023-05-13 09:38:25,327] [INFO] [utils.py:828:see_memory_usage] MA 11.93 GB Max_MA 11.93 GB CA 15.91 GB Max_CA 16 GB + 0: [2023-05-13 09:38:25,327] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.71 GB, percent = 8.3% + 0: [2023-05-13 09:38:25,436] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer + 0: [2023-05-13 09:38:25,436] [INFO] [utils.py:828:see_memory_usage] MA 12.05 GB Max_MA 12.05 GB CA 15.91 GB Max_CA 16 GB + 0: [2023-05-13 09:38:25,436] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.71 GB, percent = 8.3% + 0: [2023-05-13 09:38:25,574] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer + 0: [2023-05-13 09:38:25,575] [INFO] [utils.py:828:see_memory_usage] MA 12.05 GB Max_MA 12.05 GB CA 15.91 GB Max_CA 16 GB + 0: [2023-05-13 09:38:25,575] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.71 GB, percent = 8.3% + 0: [2023-05-13 09:38:25,575] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam + 0: [2023-05-13 09:38:25,576] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler + 0: [2023-05-13 09:38:25,576] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = + 0: [2023-05-13 09:38:25,576] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] + 0: [2023-05-13 09:38:25,577] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: + 0: [2023-05-13 09:38:25,578] [INFO] [config.py:1011:print] activation_checkpointing_config { + 0: "partition_activations": false, + 0: "contiguous_memory_optimization": false, + 0: "cpu_checkpointing": false, + 0: "number_checkpoints": null, + 0: "synchronize_checkpoint_boundary": false, + 0: "profile": false + 0: } + 0: [2023-05-13 09:38:25,578] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} + 0: [2023-05-13 09:38:25,578] [INFO] [config.py:1011:print] amp_enabled .................. False + 0: [2023-05-13 09:38:25,578] [INFO] [config.py:1011:print] amp_params ................... False + 0: [2023-05-13 09:38:25,578] [INFO] [config.py:1011:print] autotuning_config ............ { + 0: "enabled": false, + 0: "start_step": null, + 0: "end_step": null, + 0: "metric_path": null, + 0: "arg_mappings": null, + 0: "metric": "throughput", + 0: "model_info": null, + 0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", + 0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", + 0: "overwrite": true, + 0: "fast": true, + 0: "start_profile_step": 3, + 0: "end_profile_step": 5, + 0: "tuner_type": "gridsearch", + 0: "tuner_early_stopping": 5, + 0: "tuner_num_trials": 50, + 0: "model_info_path": null, + 0: "mp_size": 1, + 0: "max_train_batch_size": null, + 0: "min_train_batch_size": 1, + 0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, + 0: "min_train_micro_batch_size_per_gpu": 1, + 0: "num_tuning_micro_batch_sizes": 3 + 0: } + 0: [2023-05-13 09:38:25,578] [INFO] [config.py:1011:print] bfloat16_enabled ............. True + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] comms_config ................. + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] communication_data_type ...... None + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa + 0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] curriculum_enabled ........... False + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] curriculum_params ............ False + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] dataloader_drop_last ......... False + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] disable_allgather ............ False + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] dump_state ................... False + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 + 0: [2023-05-13 09:38:25,579] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] elasticity_enabled ........... False + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] flops_profiler_config ........ { + 0: "enabled": false, + 0: "profile_step": 1, + 0: "module_depth": -1, + 0: "top_modules": 1, + 0: "detailed": true, + 0: "output_file": null + 0: } + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] fp16_auto_cast ............... None + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] fp16_enabled ................. False + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] global_rank .................. 0 + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 4 + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] load_universal_checkpoint .... False + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] loss_scale ................... 1.0 + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] memory_breakdown ............. False + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] monitor_config ............... + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] nebula_config ................ { + 0: "enabled": false, + 0: "persistent_storage_path": null, + 0: "persistent_time_interval": 100, + 0: "num_of_version_in_retention": 2, + 0: "enable_nebula_load": true, + 0: "load_path": null + 0: } + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False + 0: [2023-05-13 09:38:25,580] [INFO] [config.py:1011:print] optimizer_name ............... None + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] optimizer_params ............. None + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] pld_enabled .................. False + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] pld_params ................... False + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] prescale_gradients ........... False + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] scheduler_name ............... None + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] scheduler_params ............. None + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] sparse_attention ............. None + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] steps_per_print .............. 2000 + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] train_batch_size ............. 1024 + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 2 + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] use_node_local_storage ....... False + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] world_size ................... 128 + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] zero_enabled ................. False + 0: [2023-05-13 09:38:25,581] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 + 0: [2023-05-13 09:38:25,582] [INFO] [config.py:996:print_user_config] json = { + 0: "train_micro_batch_size_per_gpu": 2, + 0: "train_batch_size": 1.024000e+03, + 0: "gradient_clipping": 1.0, + 0: "zero_optimization": { + 0: "stage": 0 + 0: }, + 0: "bf16": { + 0: "enabled": true + 0: }, + 0: "steps_per_print": 2.000000e+03, + 0: "wall_clock_breakdown": false + 0: } + 0: Time to load utils op: 0.0006022453308105469 seconds + 0: [2023-05-13 09:38:25,582] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=4 micro_batch_size=2 + 0: [2023-05-13 09:38:25,612] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=43 [0, 43) STAGE_PARAMS=2123243520 (2123.244M) TOTAL_PARAMS=4246487040 (4246.487M) UNIQUE_PARAMS=4246487040 (4246.487M) + 0: [2023-05-13 09:38:25,612] [INFO] [engine.py:145:__init__] RANK=1 STAGE=0 LAYERS=43 [0, 43) STAGE_PARAMS=2123243520 (2123.244M) TOTAL_PARAMS=4246487040 (4246.487M) UNIQUE_PARAMS=4246487040 (4246.487M) +16: [2023-05-13 09:38:28,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:28,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:28,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:28,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:28,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +16: [2023-05-13 09:38:28,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +16: [2023-05-13 09:38:28,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +16: [2023-05-13 09:38:28,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +16: [2023-05-13 09:38:28,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +11: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 6: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 6: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 6: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 6: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 1: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 1: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 1: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +11: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 8: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 7: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 7: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +17: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +29: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +24: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +29: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +29: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 9: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +30: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +30: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +21: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +15: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +11: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +30: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:28,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +30: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 9: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +21: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +21: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +21: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 7: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +11: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +26: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +21: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +21: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 7: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +11: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +29: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +24: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +26: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +26: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +26: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +29: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +29: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +12: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 4: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 4: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +25: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +27: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +12: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +20: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +10: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 4: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 2: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +22: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 0: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 2: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +22: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 0: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 0: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 4: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +25: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +27: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +25: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +23: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +25: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +27: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 2: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 2: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +14: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +14: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +14: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +20: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +20: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +27: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +27: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +14: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +27: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +13: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +14: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:28,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 0: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +23: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +13: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +23: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +13: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +13: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +14: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +23: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +14: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +19: [2023-05-13 09:38:28,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt... +19: [2023-05-13 09:38:28,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:28,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:28,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:28,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:28,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +26: [2023-05-13 09:38:28,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +26: [2023-05-13 09:38:28,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +26: [2023-05-13 09:38:28,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +26: [2023-05-13 09:38:28,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:28,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:28,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:28,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +16: [2023-05-13 09:38:28,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +16: [2023-05-13 09:38:28,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +16: [2023-05-13 09:38:28,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +16: [2023-05-13 09:38:28,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:28,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:28,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:28,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:28,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +26: [2023-05-13 09:38:28,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:28,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +30: [2023-05-13 09:38:28,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +30: [2023-05-13 09:38:28,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +30: [2023-05-13 09:38:28,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:28,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:28,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:28,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:28,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:28,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 8: [2023-05-13 09:38:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 8: [2023-05-13 09:38:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:28,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 5: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 5: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 5: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:28,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:28,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 5: [2023-05-13 09:38:28,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:28,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:28,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:28,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:28,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:28,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:28,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:28,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:28,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +18: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +18: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +18: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +18: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 1: [2023-05-13 09:38:28,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:28,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 3: [2023-05-13 09:38:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +10: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 1: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 3: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 1: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +10: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 3: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +10: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 9: [2023-05-13 09:38:28,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 1: [2023-05-13 09:38:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 9: [2023-05-13 09:38:28,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +10: [2023-05-13 09:38:28,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:28,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 9: [2023-05-13 09:38:28,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 9: [2023-05-13 09:38:28,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +30: [2023-05-13 09:38:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:28,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:28,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:28,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:28,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:28,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +30: [2023-05-13 09:38:28,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:28,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +23: [2023-05-13 09:38:28,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +30: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +31: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +31: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +19: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +17: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +17: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +17: [2023-05-13 09:38:28,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +19: [2023-05-13 09:38:28,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +17: [2023-05-13 09:38:28,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 0: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +13: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +19: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:28,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +13: [2023-05-13 09:38:28,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:28,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:28,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt... + 0: [2023-05-13 09:38:28,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:28,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:28,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:28,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:28,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:28,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:28,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:28,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:28,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:28,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:28,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:28,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:28,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:28,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:28,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:28,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:28,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:28,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:28,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:28,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:28,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:28,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:28,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:28,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:28,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:28,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:28,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:28,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:28,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:28,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:28,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:28,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:28,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:28,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:28,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:28,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:28,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:28,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:28,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:28,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:28,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:28,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:28,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:28,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:28,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +28: [2023-05-13 09:38:29,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +28: [2023-05-13 09:38:29,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +28: [2023-05-13 09:38:29,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +28: [2023-05-13 09:38:29,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:29,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:29,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:29,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +11: [2023-05-13 09:38:29,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +18: [2023-05-13 09:38:29,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +18: [2023-05-13 09:38:29,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +11: [2023-05-13 09:38:29,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +18: [2023-05-13 09:38:29,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +24: [2023-05-13 09:38:29,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:29,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +24: [2023-05-13 09:38:29,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:29,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:29,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:29,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:29,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 3: [2023-05-13 09:38:29,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 3: [2023-05-13 09:38:29,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:29,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +28: [2023-05-13 09:38:29,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +26: [2023-05-13 09:38:29,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +26: [2023-05-13 09:38:29,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +27: [2023-05-13 09:38:29,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +28: [2023-05-13 09:38:29,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +27: [2023-05-13 09:38:29,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +14: [2023-05-13 09:38:29,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +14: [2023-05-13 09:38:29,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +14: [2023-05-13 09:38:29,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:29,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:29,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:29,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:29,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:29,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +29: [2023-05-13 09:38:29,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +10: [2023-05-13 09:38:29,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +10: [2023-05-13 09:38:29,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +10: [2023-05-13 09:38:29,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:29,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:29,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +26: [2023-05-13 09:38:29,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +13: [2023-05-13 09:38:29,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +13: [2023-05-13 09:38:29,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:29,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +26: [2023-05-13 09:38:29,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +16: [2023-05-13 09:38:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +16: [2023-05-13 09:38:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +14: [2023-05-13 09:38:29,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:29,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +27: [2023-05-13 09:38:29,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +27: [2023-05-13 09:38:29,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +10: [2023-05-13 09:38:29,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +17: [2023-05-13 09:38:29,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +17: [2023-05-13 09:38:29,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +17: [2023-05-13 09:38:29,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:29,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 6: [2023-05-13 09:38:29,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 6: [2023-05-13 09:38:29,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +29: [2023-05-13 09:38:29,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 6: [2023-05-13 09:38:29,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:29,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:29,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:29,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 6: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +14: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:29,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +13: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +12: [2023-05-13 09:38:29,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:29,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +12: [2023-05-13 09:38:29,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +12: [2023-05-13 09:38:29,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +18: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +18: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:29,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +17: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:29,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +17: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +22: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +14: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 9: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +14: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +13: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 9: [2023-05-13 09:38:29,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +31: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:29,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 0: [2023-05-13 09:38:29,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:29,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:29,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:29,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +26: [2023-05-13 09:38:29,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +26: [2023-05-13 09:38:29,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:29,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +26: [2023-05-13 09:38:29,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:29,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +29: [2023-05-13 09:38:29,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +17: [2023-05-13 09:38:29,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +14: [2023-05-13 09:38:29,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:29,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:29,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +14: [2023-05-13 09:38:29,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +14: [2023-05-13 09:38:29,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:29,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:29,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +12: [2023-05-13 09:38:29,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 0: [2023-05-13 09:38:29,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +24: [2023-05-13 09:38:29,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 0: [2023-05-13 09:38:29,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +24: [2023-05-13 09:38:29,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:29,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +21: [2023-05-13 09:38:29,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:29,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +24: [2023-05-13 09:38:29,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:29,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:29,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:29,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +23: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +23: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +22: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +27: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +29: [2023-05-13 09:38:29,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 1: [2023-05-13 09:38:29,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 1: [2023-05-13 09:38:29,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +27: [2023-05-13 09:38:29,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +27: [2023-05-13 09:38:29,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 1: [2023-05-13 09:38:29,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +20: [2023-05-13 09:38:29,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +20: [2023-05-13 09:38:29,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:29,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:29,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +27: [2023-05-13 09:38:29,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +27: [2023-05-13 09:38:29,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +23: [2023-05-13 09:38:29,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +31: [2023-05-13 09:38:29,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +29: [2023-05-13 09:38:29,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:29,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +13: [2023-05-13 09:38:29,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:29,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_01-model_01-model_states.pt. +17: [2023-05-13 09:38:29,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:29,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:29,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +17: [2023-05-13 09:38:29,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +17: [2023-05-13 09:38:29,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +17: [2023-05-13 09:38:29,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:29,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:29,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +23: [2023-05-13 09:38:29,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:29,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:29,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +30: [2023-05-13 09:38:29,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +21: [2023-05-13 09:38:29,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +12: [2023-05-13 09:38:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:29,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:29,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +21: [2023-05-13 09:38:29,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:29,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +10: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +10: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:29,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:29,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +25: [2023-05-13 09:38:29,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +10: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:29,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:29,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:29,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:29,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:29,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:29,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +25: [2023-05-13 09:38:29,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +12: [2023-05-13 09:38:29,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 1: [2023-05-13 09:38:29,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +11: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +11: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:29,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +12: [2023-05-13 09:38:29,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +23: [2023-05-13 09:38:29,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +24: [2023-05-13 09:38:29,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:29,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +23: [2023-05-13 09:38:29,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +23: [2023-05-13 09:38:29,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:29,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:29,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:29,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:29,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:29,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:29,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:29,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:29,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:29,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +24: [2023-05-13 09:38:29,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +24: [2023-05-13 09:38:29,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +24: [2023-05-13 09:38:29,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +29: [2023-05-13 09:38:29,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:29,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:29,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:29,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:29,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:29,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:29,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:29,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:29,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:29,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:29,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +13: [2023-05-13 09:38:29,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +13: [2023-05-13 09:38:29,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +13: [2023-05-13 09:38:29,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:29,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:29,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:29,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:29,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +25: [2023-05-13 09:38:29,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +25: [2023-05-13 09:38:29,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +25: [2023-05-13 09:38:29,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +25: [2023-05-13 09:38:29,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:29,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:29,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:29,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:29,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:29,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +26: [2023-05-13 09:38:29,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +26: [2023-05-13 09:38:29,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +26: [2023-05-13 09:38:29,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +26: [2023-05-13 09:38:29,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +29: [2023-05-13 09:38:29,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +29: [2023-05-13 09:38:29,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +29: [2023-05-13 09:38:29,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +29: [2023-05-13 09:38:29,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +13: [2023-05-13 09:38:29,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:29,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:29,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:29,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:29,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:29,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:29,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:29,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:29,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:29,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:29,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:29,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +19: [2023-05-13 09:38:29,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +19: [2023-05-13 09:38:29,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +19: [2023-05-13 09:38:29,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:29,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:29,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:29,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +15: [2023-05-13 09:38:29,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +15: [2023-05-13 09:38:29,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +17: [2023-05-13 09:38:29,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:29,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:29,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:29,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:29,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:29,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:29,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:29,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:29,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +31: [2023-05-13 09:38:29,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +31: [2023-05-13 09:38:29,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +29: [2023-05-13 09:38:29,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +31: [2023-05-13 09:38:29,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 6: [2023-05-13 09:38:29,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:29,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:29,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:29,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:29,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:29,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:29,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:29,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:29,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:29,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:29,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 3: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:29,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 3: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:29,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +27: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +27: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +27: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +27: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +16: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +20: [2023-05-13 09:38:29,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +20: [2023-05-13 09:38:29,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +16: [2023-05-13 09:38:29,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +16: [2023-05-13 09:38:29,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:29,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +16: [2023-05-13 09:38:29,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:29,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +17: [2023-05-13 09:38:29,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +18: [2023-05-13 09:38:29,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +18: [2023-05-13 09:38:29,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +17: [2023-05-13 09:38:29,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +18: [2023-05-13 09:38:29,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +12: [2023-05-13 09:38:29,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +12: [2023-05-13 09:38:29,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +20: [2023-05-13 09:38:29,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:29,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 1: [2023-05-13 09:38:29,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:29,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:29,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +11: [2023-05-13 09:38:29,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:29,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:29,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +21: [2023-05-13 09:38:29,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +21: [2023-05-13 09:38:29,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:29,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +17: [2023-05-13 09:38:29,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:29,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:29,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:29,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:29,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:29,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:29,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:29,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:29,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:29,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:29,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:29,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:29,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:29,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:29,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:29,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +18: [2023-05-13 09:38:29,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:29,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:29,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:29,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:29,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +17: [2023-05-13 09:38:29,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +14: [2023-05-13 09:38:29,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:29,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:29,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:29,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:29,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:29,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:29,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:29,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +27: [2023-05-13 09:38:29,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +27: [2023-05-13 09:38:29,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +14: [2023-05-13 09:38:29,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +21: [2023-05-13 09:38:29,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:29,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +27: [2023-05-13 09:38:29,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:29,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:29,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +12: [2023-05-13 09:38:29,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:29,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:29,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:29,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:29,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +14: [2023-05-13 09:38:29,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +12: [2023-05-13 09:38:29,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +12: [2023-05-13 09:38:29,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:29,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:29,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +26: [2023-05-13 09:38:29,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +26: [2023-05-13 09:38:29,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +26: [2023-05-13 09:38:29,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +26: [2023-05-13 09:38:29,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +26: [2023-05-13 09:38:29,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +12: [2023-05-13 09:38:29,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:29,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:29,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:29,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:29,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_03-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +23: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +23: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +21: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +21: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +21: [2023-05-13 09:38:29,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:29,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +21: [2023-05-13 09:38:29,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:29,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 4: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 4: [2023-05-13 09:38:29,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +13: [2023-05-13 09:38:29,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +13: [2023-05-13 09:38:29,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:29,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:29,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:29,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:29,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:29,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 4: [2023-05-13 09:38:29,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +21: [2023-05-13 09:38:29,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 4: [2023-05-13 09:38:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +25: [2023-05-13 09:38:29,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +25: [2023-05-13 09:38:29,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:29,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:29,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:29,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:29,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:29,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:29,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:29,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 1: [2023-05-13 09:38:29,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 1: [2023-05-13 09:38:29,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 1: [2023-05-13 09:38:29,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 1: [2023-05-13 09:38:29,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:29,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +31: [2023-05-13 09:38:29,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +31: [2023-05-13 09:38:29,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:29,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:29,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:29,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +13: [2023-05-13 09:38:29,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:29,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:29,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:29,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +25: [2023-05-13 09:38:29,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:29,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:29,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:29,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:29,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:29,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:29,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:29,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:29,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:29,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:29,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:29,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +11: [2023-05-13 09:38:29,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +11: [2023-05-13 09:38:29,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +11: [2023-05-13 09:38:29,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt... +31: [2023-05-13 09:38:29,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:29,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:29,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:29,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:29,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:29,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:29,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:29,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:29,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:29,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:29,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:29,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:29,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:29,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:29,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:29,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +18: [2023-05-13 09:38:29,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +18: [2023-05-13 09:38:29,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +18: [2023-05-13 09:38:29,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +18: [2023-05-13 09:38:29,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:29,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:29,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:29,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:29,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:29,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:29,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:29,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:30,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:30,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:30,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:30,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:30,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +10: [2023-05-13 09:38:30,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +10: [2023-05-13 09:38:30,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +10: [2023-05-13 09:38:30,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +10: [2023-05-13 09:38:30,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:30,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:30,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:30,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:30,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +30: [2023-05-13 09:38:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +30: [2023-05-13 09:38:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +30: [2023-05-13 09:38:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +22: [2023-05-13 09:38:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +22: [2023-05-13 09:38:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +22: [2023-05-13 09:38:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +22: [2023-05-13 09:38:30,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:30,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:30,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:30,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:30,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:30,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:30,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +10: [2023-05-13 09:38:30,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +10: [2023-05-13 09:38:30,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 3: [2023-05-13 09:38:30,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 3: [2023-05-13 09:38:30,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +10: [2023-05-13 09:38:30,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:30,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 1: [2023-05-13 09:38:30,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +22: [2023-05-13 09:38:30,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 1: [2023-05-13 09:38:30,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:30,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:30,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:30,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:30,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:30,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:30,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:30,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:30,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:30,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +13: [2023-05-13 09:38:30,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +13: [2023-05-13 09:38:30,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +13: [2023-05-13 09:38:30,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:30,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:30,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:30,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:30,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +13: [2023-05-13 09:38:30,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:30,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:30,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:30,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:30,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:30,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:30,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:30,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +19: [2023-05-13 09:38:30,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +19: [2023-05-13 09:38:30,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +21: [2023-05-13 09:38:30,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +21: [2023-05-13 09:38:30,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +19: [2023-05-13 09:38:30,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +20: [2023-05-13 09:38:30,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +21: [2023-05-13 09:38:30,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +21: [2023-05-13 09:38:30,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +20: [2023-05-13 09:38:30,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +20: [2023-05-13 09:38:30,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +20: [2023-05-13 09:38:30,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:30,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:30,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:30,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:30,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:30,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:30,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:30,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:30,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 5: [2023-05-13 09:38:30,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 5: [2023-05-13 09:38:30,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 5: [2023-05-13 09:38:30,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 5: [2023-05-13 09:38:30,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +21: [2023-05-13 09:38:30,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:30,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:30,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:30,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:30,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:30,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:30,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:30,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:30,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:30,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:30,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:30,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:30,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:30,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:30,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:30,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:30,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:30,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:30,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:30,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:30,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:30,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:30,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:30,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:30,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:30,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:30,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:30,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +23: [2023-05-13 09:38:30,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +23: [2023-05-13 09:38:30,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +23: [2023-05-13 09:38:30,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:30,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:30,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:30,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:30,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:30,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:30,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +26: [2023-05-13 09:38:30,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:30,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:30,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:30,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:30,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:30,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:30,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:30,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:30,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +11: [2023-05-13 09:38:30,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +11: [2023-05-13 09:38:30,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +26: [2023-05-13 09:38:30,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:30,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +25: [2023-05-13 09:38:30,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:30,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +25: [2023-05-13 09:38:30,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_04-model_01-model_states.pt. +11: [2023-05-13 09:38:30,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:30,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:30,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +29: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +17: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +29: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:30,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:30,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:30,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 5: [2023-05-13 09:38:30,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 5: [2023-05-13 09:38:30,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 5: [2023-05-13 09:38:30,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 5: [2023-05-13 09:38:30,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:30,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:30,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:30,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:30,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:30,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +29: [2023-05-13 09:38:30,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +29: [2023-05-13 09:38:30,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +29: [2023-05-13 09:38:30,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:30,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:30,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:30,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:30,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:30,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:30,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:30,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:30,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:30,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:30,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:30,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +27: [2023-05-13 09:38:30,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +27: [2023-05-13 09:38:30,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +27: [2023-05-13 09:38:30,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +27: [2023-05-13 09:38:30,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:30,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:30,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +15: [2023-05-13 09:38:30,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +15: [2023-05-13 09:38:30,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +29: [2023-05-13 09:38:30,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:30,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +15: [2023-05-13 09:38:30,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:30,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:30,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:30,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:30,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:30,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:30,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:30,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:30,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +22: [2023-05-13 09:38:30,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +22: [2023-05-13 09:38:30,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:30,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:30,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:30,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:30,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:30,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:30,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +20: [2023-05-13 09:38:30,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +20: [2023-05-13 09:38:30,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:30,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:30,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:30,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 6: [2023-05-13 09:38:30,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:30,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 6: [2023-05-13 09:38:30,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:30,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +15: [2023-05-13 09:38:30,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 6: [2023-05-13 09:38:30,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:30,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 6: [2023-05-13 09:38:30,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:30,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:30,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 4: [2023-05-13 09:38:30,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 4: [2023-05-13 09:38:30,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:30,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 4: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:30,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 4: [2023-05-13 09:38:30,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:30,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:30,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:30,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:30,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:30,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:30,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:30,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 5: [2023-05-13 09:38:30,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 5: [2023-05-13 09:38:30,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 5: [2023-05-13 09:38:30,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 6: [2023-05-13 09:38:30,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:30,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:30,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:30,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:30,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:30,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:30,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 6: [2023-05-13 09:38:30,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 4: [2023-05-13 09:38:30,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:30,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:30,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:30,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +30: [2023-05-13 09:38:30,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +30: [2023-05-13 09:38:30,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +30: [2023-05-13 09:38:30,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:30,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:30,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:30,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:30,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:30,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:30,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:30,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:30,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +23: [2023-05-13 09:38:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +23: [2023-05-13 09:38:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +23: [2023-05-13 09:38:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:30,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:30,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:30,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +13: [2023-05-13 09:38:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 7: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 2: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 2: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:30,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 7: [2023-05-13 09:38:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:30,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 7: [2023-05-13 09:38:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +13: [2023-05-13 09:38:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 8: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:30,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:30,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 8: [2023-05-13 09:38:30,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +26: [2023-05-13 09:38:30,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +26: [2023-05-13 09:38:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +26: [2023-05-13 09:38:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +26: [2023-05-13 09:38:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +26: [2023-05-13 09:38:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:30,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:30,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:30,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:30,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:30,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 8: [2023-05-13 09:38:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:30,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:30,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +24: [2023-05-13 09:38:30,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:30,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:30,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:30,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:30,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:30,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:30,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:30,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:30,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:30,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:30,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:30,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 0: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 2: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +14: [2023-05-13 09:38:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +14: [2023-05-13 09:38:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +14: [2023-05-13 09:38:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +14: [2023-05-13 09:38:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +10: [2023-05-13 09:38:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +10: [2023-05-13 09:38:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 0: [2023-05-13 09:38:30,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 0: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 0: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:30,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:30,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:30,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:30,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:30,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:30,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:30,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:30,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:30,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:30,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:30,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:30,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +22: [2023-05-13 09:38:30,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +22: [2023-05-13 09:38:30,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +22: [2023-05-13 09:38:30,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:30,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:30,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:30,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:30,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:30,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:30,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:30,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:30,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:30,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 8: [2023-05-13 09:38:30,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +17: [2023-05-13 09:38:30,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +17: [2023-05-13 09:38:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:30,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:30,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +29: [2023-05-13 09:38:30,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +29: [2023-05-13 09:38:30,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +29: [2023-05-13 09:38:30,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +22: [2023-05-13 09:38:30,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:30,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:30,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:30,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:30,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:30,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:30,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:30,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:30,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:30,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:30,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:30,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:30,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:30,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:30,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:30,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:30,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:30,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:30,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:30,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:30,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:30,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:30,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:31,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:31,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:31,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:31,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +12: [2023-05-13 09:38:31,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +12: [2023-05-13 09:38:31,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +12: [2023-05-13 09:38:31,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +15: [2023-05-13 09:38:31,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +31: [2023-05-13 09:38:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:31,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:31,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:31,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:31,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +31: [2023-05-13 09:38:31,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +31: [2023-05-13 09:38:31,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +31: [2023-05-13 09:38:31,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:31,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +31: [2023-05-13 09:38:31,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:31,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:31,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +15: [2023-05-13 09:38:31,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:31,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:31,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:31,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:31,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:31,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:31,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:31,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:31,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:31,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:31,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:31,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:31,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:31,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:31,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +31: [2023-05-13 09:38:31,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +31: [2023-05-13 09:38:31,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +25: [2023-05-13 09:38:31,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:31,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:31,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +16: [2023-05-13 09:38:31,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +16: [2023-05-13 09:38:31,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +16: [2023-05-13 09:38:31,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +16: [2023-05-13 09:38:31,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:31,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:31,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:31,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +17: [2023-05-13 09:38:31,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:31,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +29: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:31,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:31,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:31,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:31,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:31,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:31,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:31,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:31,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:31,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:31,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:31,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:31,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:31,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:31,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:31,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:31,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:31,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:31,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:31,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:31,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:31,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:31,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:31,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +15: [2023-05-13 09:38:31,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:31,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:31,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:31,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:31,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +23: [2023-05-13 09:38:31,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +23: [2023-05-13 09:38:31,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +16: [2023-05-13 09:38:31,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:31,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:31,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:31,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +16: [2023-05-13 09:38:31,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +16: [2023-05-13 09:38:31,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:31,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:31,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:31,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:31,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:31,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:31,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:31,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:31,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +13: [2023-05-13 09:38:31,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:31,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:31,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +13: [2023-05-13 09:38:31,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:31,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:31,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:31,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:31,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:31,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:31,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:31,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:31,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:31,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:31,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:31,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:31,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:31,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:31,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:31,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:31,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:31,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:31,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:31,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:31,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +15: [2023-05-13 09:38:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:31,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +15: [2023-05-13 09:38:31,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:31,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:31,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:31,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:31,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:31,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:31,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:31,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:31,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:31,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:31,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +23: [2023-05-13 09:38:31,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +30: [2023-05-13 09:38:31,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:31,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:31,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +11: [2023-05-13 09:38:31,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +25: [2023-05-13 09:38:31,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:31,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:31,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:31,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:31,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +23: [2023-05-13 09:38:31,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +23: [2023-05-13 09:38:31,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +30: [2023-05-13 09:38:31,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:31,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:31,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:31,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:31,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:31,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:31,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:31,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +26: [2023-05-13 09:38:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +11: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:31,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 2: [2023-05-13 09:38:31,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +18: [2023-05-13 09:38:31,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +24: [2023-05-13 09:38:31,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +18: [2023-05-13 09:38:31,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:31,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:31,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:31,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:31,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:31,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +19: [2023-05-13 09:38:31,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +19: [2023-05-13 09:38:31,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +19: [2023-05-13 09:38:31,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +26: [2023-05-13 09:38:31,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +26: [2023-05-13 09:38:31,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +26: [2023-05-13 09:38:31,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:31,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:31,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:31,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:31,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:31,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:31,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:31,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:31,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:31,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:31,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:31,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:31,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:31,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:31,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:31,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:31,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +28: [2023-05-13 09:38:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +28: [2023-05-13 09:38:31,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +28: [2023-05-13 09:38:31,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +28: [2023-05-13 09:38:31,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +18: [2023-05-13 09:38:31,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:31,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:31,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:31,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:31,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:31,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:31,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:31,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:31,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:31,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:31,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:31,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:31,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:31,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +24: [2023-05-13 09:38:31,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:31,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:31,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +10: [2023-05-13 09:38:31,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +19: [2023-05-13 09:38:31,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:31,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:31,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +19: [2023-05-13 09:38:31,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:31,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +26: [2023-05-13 09:38:31,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +19: [2023-05-13 09:38:31,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +26: [2023-05-13 09:38:31,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +10: [2023-05-13 09:38:31,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +18: [2023-05-13 09:38:31,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +26: [2023-05-13 09:38:31,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:31,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:31,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:31,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +26: [2023-05-13 09:38:31,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +19: [2023-05-13 09:38:31,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +14: [2023-05-13 09:38:31,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +19: [2023-05-13 09:38:31,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:31,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_05-model_01-model_states.pt. +11: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +19: [2023-05-13 09:38:31,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:31,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:31,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:31,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:31,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:31,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +24: [2023-05-13 09:38:31,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:31,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:31,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:31,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +24: [2023-05-13 09:38:31,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +24: [2023-05-13 09:38:31,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:31,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +10: [2023-05-13 09:38:31,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +31: [2023-05-13 09:38:31,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +31: [2023-05-13 09:38:31,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +31: [2023-05-13 09:38:31,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:31,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +19: [2023-05-13 09:38:31,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:31,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:31,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:31,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +14: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:31,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:31,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:31,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +11: [2023-05-13 09:38:31,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +19: [2023-05-13 09:38:31,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:31,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:31,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:31,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:31,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:31,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:31,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +15: [2023-05-13 09:38:31,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +15: [2023-05-13 09:38:31,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +15: [2023-05-13 09:38:31,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:31,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:31,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:31,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:31,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +15: [2023-05-13 09:38:31,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:31,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:31,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +13: [2023-05-13 09:38:31,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +13: [2023-05-13 09:38:31,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +13: [2023-05-13 09:38:31,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +15: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +20: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +20: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +20: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +25: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +25: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +20: [2023-05-13 09:38:31,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +15: [2023-05-13 09:38:31,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:31,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:31,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:31,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:31,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:31,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:31,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:31,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:31,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:31,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:31,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:31,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:31,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:31,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:31,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:31,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:31,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:31,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:31,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:31,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +18: [2023-05-13 09:38:31,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +18: [2023-05-13 09:38:31,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +18: [2023-05-13 09:38:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +19: [2023-05-13 09:38:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +19: [2023-05-13 09:38:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +19: [2023-05-13 09:38:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +19: [2023-05-13 09:38:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +24: [2023-05-13 09:38:31,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +24: [2023-05-13 09:38:31,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +24: [2023-05-13 09:38:31,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +24: [2023-05-13 09:38:31,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +18: [2023-05-13 09:38:31,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:31,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:31,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:31,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:31,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:31,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:31,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:31,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:31,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +11: [2023-05-13 09:38:31,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +11: [2023-05-13 09:38:31,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +11: [2023-05-13 09:38:31,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +11: [2023-05-13 09:38:31,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:31,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:31,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:31,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:31,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:31,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +22: [2023-05-13 09:38:31,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +22: [2023-05-13 09:38:31,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:31,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:31,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +22: [2023-05-13 09:38:31,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +29: [2023-05-13 09:38:31,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:31,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +29: [2023-05-13 09:38:31,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +29: [2023-05-13 09:38:31,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:31,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:31,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:31,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:31,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:31,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:31,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:31,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:31,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +29: [2023-05-13 09:38:31,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:31,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:31,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:31,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:31,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:31,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:31,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:31,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:31,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:31,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:31,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:31,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:31,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +17: [2023-05-13 09:38:31,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +17: [2023-05-13 09:38:31,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +17: [2023-05-13 09:38:31,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:31,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +17: [2023-05-13 09:38:31,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:31,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:31,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +29: [2023-05-13 09:38:31,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:31,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:31,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:31,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:31,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:31,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:31,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:31,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:31,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:31,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:31,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:31,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:31,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:31,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:31,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:31,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:31,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:31,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:31,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:31,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +14: [2023-05-13 09:38:31,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +14: [2023-05-13 09:38:31,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +14: [2023-05-13 09:38:31,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +14: [2023-05-13 09:38:31,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +26: [2023-05-13 09:38:31,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +26: [2023-05-13 09:38:31,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +26: [2023-05-13 09:38:31,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +26: [2023-05-13 09:38:31,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:31,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:31,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:31,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:31,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:31,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:31,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:31,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:31,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:31,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:31,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:31,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:31,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:31,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:31,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:31,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:31,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:31,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:31,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:31,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:32,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:32,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +12: [2023-05-13 09:38:32,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +12: [2023-05-13 09:38:32,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +25: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +17: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +17: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +21: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:32,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +20: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +27: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +20: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +20: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:32,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +27: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +27: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +27: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +27: [2023-05-13 09:38:32,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +28: [2023-05-13 09:38:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:32,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +28: [2023-05-13 09:38:32,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:32,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +28: [2023-05-13 09:38:32,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:32,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +28: [2023-05-13 09:38:32,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:32,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:32,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:32,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:32,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:32,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:32,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:32,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:32,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:32,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:32,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:32,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:32,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:32,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:32,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:32,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:32,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:32,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:32,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:32,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:32,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:32,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:32,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:32,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:32,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:32,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:32,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:32,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:32,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:32,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:32,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:32,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:32,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:32,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +28: [2023-05-13 09:38:32,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +15: [2023-05-13 09:38:32,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:32,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:32,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +28: [2023-05-13 09:38:32,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:32,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:32,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:32,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +15: [2023-05-13 09:38:32,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:32,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +15: [2023-05-13 09:38:32,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:32,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +15: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:32,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:32,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:32,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +29: [2023-05-13 09:38:32,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +29: [2023-05-13 09:38:32,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +29: [2023-05-13 09:38:32,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:32,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +14: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +14: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +14: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +14: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +14: [2023-05-13 09:38:32,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +30: [2023-05-13 09:38:32,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:32,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +30: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +30: [2023-05-13 09:38:32,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +30: [2023-05-13 09:38:32,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +29: [2023-05-13 09:38:32,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:32,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +29: [2023-05-13 09:38:32,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:32,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:32,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:32,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:32,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:32,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:32,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:32,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:32,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:32,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:32,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:32,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:32,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:32,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:32,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +18: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +18: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +26: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +19: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +19: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +19: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +19: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +26: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +26: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +30: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:32,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:32,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:32,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:32,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:32,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:32,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:32,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:32,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:32,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:32,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:32,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:32,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:32,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:32,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:32,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:32,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +12: [2023-05-13 09:38:32,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +12: [2023-05-13 09:38:32,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +12: [2023-05-13 09:38:32,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +12: [2023-05-13 09:38:32,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:32,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:32,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:32,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:32,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:32,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:32,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:32,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:32,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:32,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:32,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:32,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:32,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:32,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:32,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:32,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +12: [2023-05-13 09:38:32,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 8: [2023-05-13 09:38:32,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +29: [2023-05-13 09:38:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +29: [2023-05-13 09:38:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +29: [2023-05-13 09:38:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +29: [2023-05-13 09:38:32,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 8: [2023-05-13 09:38:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 8: [2023-05-13 09:38:32,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +13: [2023-05-13 09:38:32,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:32,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:32,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:32,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:32,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:32,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:32,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:32,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:32,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:32,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:32,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:32,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:32,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:32,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +19: [2023-05-13 09:38:32,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:32,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:32,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:32,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +19: [2023-05-13 09:38:32,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:32,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:32,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +23: [2023-05-13 09:38:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +23: [2023-05-13 09:38:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +25: [2023-05-13 09:38:32,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +25: [2023-05-13 09:38:32,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +23: [2023-05-13 09:38:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +19: [2023-05-13 09:38:32,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:32,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:32,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:32,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:32,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +14: [2023-05-13 09:38:32,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:32,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:32,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:32,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:32,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:32,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:32,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:32,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +21: [2023-05-13 09:38:32,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:32,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +23: [2023-05-13 09:38:32,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +27: [2023-05-13 09:38:32,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:32,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:32,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +29: [2023-05-13 09:38:32,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +21: [2023-05-13 09:38:32,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +17: [2023-05-13 09:38:32,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +17: [2023-05-13 09:38:32,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:32,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +17: [2023-05-13 09:38:32,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:32,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:32,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:32,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 2: [2023-05-13 09:38:32,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:32,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:32,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:32,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:32,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:32,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +26: [2023-05-13 09:38:32,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +26: [2023-05-13 09:38:32,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:32,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +29: [2023-05-13 09:38:32,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +28: [2023-05-13 09:38:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +21: [2023-05-13 09:38:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +21: [2023-05-13 09:38:32,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +16: [2023-05-13 09:38:32,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +16: [2023-05-13 09:38:32,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +16: [2023-05-13 09:38:32,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +16: [2023-05-13 09:38:32,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +21: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +28: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +21: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +21: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +21: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +21: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:32,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +21: [2023-05-13 09:38:32,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:32,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:32,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +19: [2023-05-13 09:38:32,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +19: [2023-05-13 09:38:32,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +19: [2023-05-13 09:38:32,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +19: [2023-05-13 09:38:32,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:32,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:32,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:32,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:32,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:32,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:32,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +21: [2023-05-13 09:38:32,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:32,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:32,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:32,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:32,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +24: [2023-05-13 09:38:32,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:32,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:32,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +11: [2023-05-13 09:38:32,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:32,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:32,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:32,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:32,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:32,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +11: [2023-05-13 09:38:32,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:32,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:32,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:32,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:32,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:32,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:32,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:32,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:32,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:32,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:32,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:32,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:32,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:32,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:32,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:32,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +27: [2023-05-13 09:38:32,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:32,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +27: [2023-05-13 09:38:32,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:32,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +21: [2023-05-13 09:38:32,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:32,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:32,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:32,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:32,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:32,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:32,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +28: [2023-05-13 09:38:32,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +28: [2023-05-13 09:38:32,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +28: [2023-05-13 09:38:32,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:32,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:32,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +10: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +10: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +19: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +25: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +26: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +26: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +16: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +16: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +16: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +16: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +26: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. +11: [2023-05-13 09:38:32,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:32,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_07-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:32,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:32,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:32,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:32,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:32,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +24: [2023-05-13 09:38:32,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:32,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:32,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:32,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:32,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:32,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +24: [2023-05-13 09:38:32,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +28: [2023-05-13 09:38:32,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:32,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:32,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:32,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:32,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:32,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:32,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:32,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +13: [2023-05-13 09:38:32,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:32,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:32,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:32,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:32,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:32,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:32,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:32,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +11: [2023-05-13 09:38:32,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:32,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:32,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:32,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:32,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:32,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:32,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:32,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +10: [2023-05-13 09:38:32,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +10: [2023-05-13 09:38:32,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +24: [2023-05-13 09:38:32,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:32,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +24: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +13: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:32,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt... +13: [2023-05-13 09:38:32,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 8: [2023-05-13 09:38:32,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +22: [2023-05-13 09:38:32,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 8: [2023-05-13 09:38:32,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:32,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:32,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:32,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +13: [2023-05-13 09:38:32,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:32,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:32,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:32,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:32,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:32,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +29: [2023-05-13 09:38:32,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +29: [2023-05-13 09:38:32,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +29: [2023-05-13 09:38:32,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +29: [2023-05-13 09:38:32,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:32,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:32,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:32,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:32,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:32,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:32,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:32,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:32,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:32,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:32,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:32,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:32,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +18: [2023-05-13 09:38:32,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +18: [2023-05-13 09:38:32,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +18: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +27: [2023-05-13 09:38:32,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:32,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +21: [2023-05-13 09:38:32,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +21: [2023-05-13 09:38:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +21: [2023-05-13 09:38:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:32,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:32,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:32,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +29: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +29: [2023-05-13 09:38:32,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:32,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +21: [2023-05-13 09:38:32,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:32,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:32,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:32,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:32,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:32,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:32,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:32,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:32,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:32,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +21: [2023-05-13 09:38:32,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +22: [2023-05-13 09:38:32,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +22: [2023-05-13 09:38:32,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +19: [2023-05-13 09:38:32,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:32,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:32,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +19: [2023-05-13 09:38:32,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 8: [2023-05-13 09:38:32,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:32,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:32,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +30: [2023-05-13 09:38:32,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +30: [2023-05-13 09:38:32,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:32,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:32,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:32,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:32,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:32,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +12: [2023-05-13 09:38:32,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +12: [2023-05-13 09:38:32,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +12: [2023-05-13 09:38:32,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +12: [2023-05-13 09:38:32,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:32,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:32,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +20: [2023-05-13 09:38:32,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:32,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:32,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:32,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:32,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:32,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:32,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:32,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:32,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:32,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:32,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:32,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:32,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:32,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:32,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:32,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:32,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:32,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:32,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:32,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:32,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:32,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:32,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:32,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:32,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:32,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:32,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:32,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:32,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:32,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:32,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +11: [2023-05-13 09:38:32,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:32,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 2: [2023-05-13 09:38:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 2: [2023-05-13 09:38:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 2: [2023-05-13 09:38:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +11: [2023-05-13 09:38:32,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:32,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +11: [2023-05-13 09:38:32,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:32,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +17: [2023-05-13 09:38:32,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:32,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +17: [2023-05-13 09:38:32,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:32,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:32,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:32,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:32,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +27: [2023-05-13 09:38:32,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:32,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +31: [2023-05-13 09:38:32,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:32,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:32,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:32,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:32,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:32,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +11: [2023-05-13 09:38:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:32,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:32,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:32,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:32,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +18: [2023-05-13 09:38:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +18: [2023-05-13 09:38:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +17: [2023-05-13 09:38:32,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:32,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:32,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:32,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:32,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:32,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:32,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:32,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +23: [2023-05-13 09:38:32,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +23: [2023-05-13 09:38:32,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +28: [2023-05-13 09:38:32,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +28: [2023-05-13 09:38:32,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +23: [2023-05-13 09:38:32,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:32,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:32,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +28: [2023-05-13 09:38:32,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:32,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:32,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +14: [2023-05-13 09:38:32,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:32,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:32,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:32,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:32,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:32,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:32,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:32,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:32,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +14: [2023-05-13 09:38:32,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +22: [2023-05-13 09:38:32,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:32,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:32,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:32,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:32,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:32,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:32,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:32,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:32,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:32,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:32,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +28: [2023-05-13 09:38:32,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:32,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:32,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:32,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:32,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:32,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +14: [2023-05-13 09:38:32,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +18: [2023-05-13 09:38:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +18: [2023-05-13 09:38:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:32,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:32,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +23: [2023-05-13 09:38:32,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:32,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:32,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +30: [2023-05-13 09:38:32,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +30: [2023-05-13 09:38:32,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +23: [2023-05-13 09:38:32,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:32,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:32,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:32,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:32,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:32,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +30: [2023-05-13 09:38:32,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +30: [2023-05-13 09:38:32,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +30: [2023-05-13 09:38:32,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:32,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:32,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +30: [2023-05-13 09:38:32,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:32,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:32,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:32,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:32,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:32,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:32,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:32,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:32,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:32,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:32,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:32,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +23: [2023-05-13 09:38:32,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:32,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:32,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:32,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:32,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:32,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +30: [2023-05-13 09:38:32,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:32,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +24: [2023-05-13 09:38:32,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +23: [2023-05-13 09:38:32,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:32,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +20: [2023-05-13 09:38:32,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +20: [2023-05-13 09:38:32,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +20: [2023-05-13 09:38:32,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +20: [2023-05-13 09:38:32,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:32,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:32,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:32,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:32,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +27: [2023-05-13 09:38:32,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:32,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:32,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +25: [2023-05-13 09:38:32,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +27: [2023-05-13 09:38:32,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:32,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +23: [2023-05-13 09:38:32,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:32,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:32,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:32,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:32,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:32,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:33,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:33,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:33,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:33,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:33,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:33,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:33,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:33,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:33,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:33,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:33,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_08-model_01-model_states.pt. +24: [2023-05-13 09:38:33,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:33,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:33,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:33,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:33,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:33,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:33,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:33,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:33,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:33,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:33,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +18: [2023-05-13 09:38:32,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:32,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:32,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:32,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:33,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:33,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:33,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:33,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:33,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:33,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:33,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:33,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:33,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:33,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:33,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:33,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:33,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:33,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:33,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:33,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:33,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:33,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:33,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +25: [2023-05-13 09:38:33,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:33,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:33,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:33,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:33,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:33,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:33,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:33,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:33,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:33,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:33,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:33,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:33,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:33,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +23: [2023-05-13 09:38:33,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:33,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:33,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:33,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:33,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:33,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:33,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +14: [2023-05-13 09:38:33,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +14: [2023-05-13 09:38:33,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +14: [2023-05-13 09:38:33,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:33,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +25: [2023-05-13 09:38:33,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:33,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:33,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:33,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:33,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:33,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:33,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:33,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:33,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +26: [2023-05-13 09:38:33,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:33,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:33,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:33,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:33,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:33,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:33,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +25: [2023-05-13 09:38:33,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +26: [2023-05-13 09:38:33,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:33,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:33,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +26: [2023-05-13 09:38:33,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:33,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:33,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:33,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:33,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +26: [2023-05-13 09:38:33,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +25: [2023-05-13 09:38:33,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:33,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:33,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:33,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:33,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:33,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:33,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:33,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:33,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:33,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:33,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:33,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:33,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:33,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:33,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +11: [2023-05-13 09:38:33,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +28: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +28: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +28: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:33,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:33,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:33,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +26: [2023-05-13 09:38:33,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:33,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:33,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:33,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:33,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +11: [2023-05-13 09:38:33,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:33,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:33,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:33,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:33,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:33,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:33,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:33,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:33,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:33,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... +25: [2023-05-13 09:38:33,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:33,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:33,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:33,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:33,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:33,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:33,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:33,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +15: [2023-05-13 09:38:33,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +15: [2023-05-13 09:38:33,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +15: [2023-05-13 09:38:33,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +15: [2023-05-13 09:38:33,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +15: [2023-05-13 09:38:33,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:33,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:33,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:33,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:33,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:33,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:33,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:33,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:33,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:33,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:33,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:33,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:33,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:33,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:33,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:33,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:33,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:33,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:33,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:33,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:33,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:33,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:33,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:33,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:33,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:33,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:33,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:33,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:33,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:33,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:33,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:33,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:33,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +18: [2023-05-13 09:38:33,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:33,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:33,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:33,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +15: [2023-05-13 09:38:33,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +15: [2023-05-13 09:38:33,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +15: [2023-05-13 09:38:33,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:33,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +18: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +15: [2023-05-13 09:38:33,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +12: [2023-05-13 09:38:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +12: [2023-05-13 09:38:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:33,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:33,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:33,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +31: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +31: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +31: [2023-05-13 09:38:33,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:33,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +19: [2023-05-13 09:38:33,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +19: [2023-05-13 09:38:33,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:33,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:33,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:33,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:33,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:33,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:33,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +20: [2023-05-13 09:38:33,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:33,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +18: [2023-05-13 09:38:33,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +18: [2023-05-13 09:38:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +23: [2023-05-13 09:38:33,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +23: [2023-05-13 09:38:33,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +31: [2023-05-13 09:38:33,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:33,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +23: [2023-05-13 09:38:33,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:33,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +23: [2023-05-13 09:38:33,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:33,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:33,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:33,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +20: [2023-05-13 09:38:33,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:33,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:33,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:33,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:33,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:33,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:33,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:33,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:33,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:33,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:33,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:33,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:33,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:33,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:33,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:33,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:33,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:33,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:33,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:33,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:33,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:33,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:33,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:33,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:33,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:33,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:33,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:33,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:33,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:33,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:33,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:33,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +23: [2023-05-13 09:38:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:33,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:33,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:33,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:33,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:33,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +30: [2023-05-13 09:38:33,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +19: [2023-05-13 09:38:33,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:33,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +19: [2023-05-13 09:38:33,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +19: [2023-05-13 09:38:33,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +25: [2023-05-13 09:38:33,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:33,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:33,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +25: [2023-05-13 09:38:33,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:33,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:33,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:33,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:33,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:33,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:33,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +23: [2023-05-13 09:38:33,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +23: [2023-05-13 09:38:33,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:33,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +29: [2023-05-13 09:38:33,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:33,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +29: [2023-05-13 09:38:33,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:33,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +10: [2023-05-13 09:38:33,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:33,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +10: [2023-05-13 09:38:33,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:33,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:33,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:33,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:33,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:33,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:33,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:33,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:33,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:33,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +12: [2023-05-13 09:38:33,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:33,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:33,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +12: [2023-05-13 09:38:33,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +25: [2023-05-13 09:38:33,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:33,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:33,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:33,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:33,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:33,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:33,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:33,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:33,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:33,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:33,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:33,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:33,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:33,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:33,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:33,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +30: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:33,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:33,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +21: [2023-05-13 09:38:33,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:33,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +24: [2023-05-13 09:38:33,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:33,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:33,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:33,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:33,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:33,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:33,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:33,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +20: [2023-05-13 09:38:33,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:33,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +13: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:33,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +20: [2023-05-13 09:38:33,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:33,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +13: [2023-05-13 09:38:33,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:33,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:33,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +22: [2023-05-13 09:38:33,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:33,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:33,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:33,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:33,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:33,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:33,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:33,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +16: [2023-05-13 09:38:33,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:33,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +20: [2023-05-13 09:38:33,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +16: [2023-05-13 09:38:33,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:33,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +26: [2023-05-13 09:38:33,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:33,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +24: [2023-05-13 09:38:33,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:33,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:33,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:33,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +21: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:33,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:33,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:33,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:33,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:33,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:33,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:33,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:33,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:33,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:33,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:33,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:33,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +30: [2023-05-13 09:38:33,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:33,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +30: [2023-05-13 09:38:33,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:33,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:33,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +10: [2023-05-13 09:38:33,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:33,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:33,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:33,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +22: [2023-05-13 09:38:33,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:33,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:33,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +15: [2023-05-13 09:38:33,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +15: [2023-05-13 09:38:33,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +15: [2023-05-13 09:38:33,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +24: [2023-05-13 09:38:33,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:33,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:33,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:33,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +13: [2023-05-13 09:38:33,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +24: [2023-05-13 09:38:33,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:33,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:33,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +22: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +13: [2023-05-13 09:38:33,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +13: [2023-05-13 09:38:33,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:33,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:33,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:33,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:33,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:33,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:33,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:33,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +14: [2023-05-13 09:38:33,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:33,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +18: [2023-05-13 09:38:33,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:33,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:33,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +26: [2023-05-13 09:38:33,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:33,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +18: [2023-05-13 09:38:33,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:33,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +26: [2023-05-13 09:38:33,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:33,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +21: [2023-05-13 09:38:33,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:33,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:33,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +14: [2023-05-13 09:38:33,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:33,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:33,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:33,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:33,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:33,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:33,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:33,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:33,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:33,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:33,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +13: [2023-05-13 09:38:33,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:33,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:33,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:33,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_09-model_01-model_states.pt. +31: [2023-05-13 09:38:33,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:33,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:33,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +27: [2023-05-13 09:38:33,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:33,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:33,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +26: [2023-05-13 09:38:33,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:33,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:33,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +28: [2023-05-13 09:38:33,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:33,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:33,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:33,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +23: [2023-05-13 09:38:33,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +23: [2023-05-13 09:38:33,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +23: [2023-05-13 09:38:33,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +26: [2023-05-13 09:38:33,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +26: [2023-05-13 09:38:33,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:33,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:33,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +11: [2023-05-13 09:38:33,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:33,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:33,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +11: [2023-05-13 09:38:33,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:33,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +27: [2023-05-13 09:38:33,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:33,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:33,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:33,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:33,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:33,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +28: [2023-05-13 09:38:33,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:33,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:33,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +11: [2023-05-13 09:38:33,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +11: [2023-05-13 09:38:33,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +11: [2023-05-13 09:38:33,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:33,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +27: [2023-05-13 09:38:33,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +19: [2023-05-13 09:38:33,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +19: [2023-05-13 09:38:33,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:33,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +29: [2023-05-13 09:38:33,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +29: [2023-05-13 09:38:33,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +29: [2023-05-13 09:38:33,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +12: [2023-05-13 09:38:33,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +12: [2023-05-13 09:38:33,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +12: [2023-05-13 09:38:33,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:33,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +19: [2023-05-13 09:38:33,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +19: [2023-05-13 09:38:33,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +19: [2023-05-13 09:38:33,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:33,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:33,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:33,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:33,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +18: [2023-05-13 09:38:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +18: [2023-05-13 09:38:33,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +18: [2023-05-13 09:38:33,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +18: [2023-05-13 09:38:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +14: [2023-05-13 09:38:33,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +14: [2023-05-13 09:38:33,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +14: [2023-05-13 09:38:33,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:33,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:33,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:33,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:33,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:33,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +20: [2023-05-13 09:38:33,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +20: [2023-05-13 09:38:33,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +20: [2023-05-13 09:38:33,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:33,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:33,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:33,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +18: [2023-05-13 09:38:33,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +12: [2023-05-13 09:38:33,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +20: [2023-05-13 09:38:33,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:33,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +12: [2023-05-13 09:38:33,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:33,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:33,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:33,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:33,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:33,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:33,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:33,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:33,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:33,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:33,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +16: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +16: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:33,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:33,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +20: [2023-05-13 09:38:33,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:33,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +12: [2023-05-13 09:38:33,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:33,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:33,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:33,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:33,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:33,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:33,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:33,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +12: [2023-05-13 09:38:33,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +12: [2023-05-13 09:38:33,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:33,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +25: [2023-05-13 09:38:33,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +25: [2023-05-13 09:38:33,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +12: [2023-05-13 09:38:33,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +25: [2023-05-13 09:38:33,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:33,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +26: [2023-05-13 09:38:33,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +24: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +31: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:33,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:33,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +24: [2023-05-13 09:38:33,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:33,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +31: [2023-05-13 09:38:33,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:33,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:33,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +30: [2023-05-13 09:38:33,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +30: [2023-05-13 09:38:33,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:33,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:33,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:33,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:33,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:33,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +17: [2023-05-13 09:38:33,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:33,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +27: [2023-05-13 09:38:33,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +27: [2023-05-13 09:38:33,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +27: [2023-05-13 09:38:33,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:33,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:33,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +13: [2023-05-13 09:38:33,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:33,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:33,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:33,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:33,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +31: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +13: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +31: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +21: [2023-05-13 09:38:33,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +27: [2023-05-13 09:38:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +26: [2023-05-13 09:38:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +31: [2023-05-13 09:38:33,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:33,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +17: [2023-05-13 09:38:33,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +17: [2023-05-13 09:38:33,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:33,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:33,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +31: [2023-05-13 09:38:33,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:33,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:33,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +28: [2023-05-13 09:38:33,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +14: [2023-05-13 09:38:33,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:33,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +28: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +28: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +28: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +28: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +10: [2023-05-13 09:38:33,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +10: [2023-05-13 09:38:33,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +17: [2023-05-13 09:38:33,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +11: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +11: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:33,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:33,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +21: [2023-05-13 09:38:33,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:33,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:33,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:33,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:33,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:33,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:33,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +16: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +16: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +16: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:33,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:33,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:33,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:33,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +26: [2023-05-13 09:38:33,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +27: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +26: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +26: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_10-model_01-model_states.pt. +24: [2023-05-13 09:38:33,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:33,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:33,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +27: [2023-05-13 09:38:33,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +27: [2023-05-13 09:38:33,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +14: [2023-05-13 09:38:33,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:33,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:33,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:33,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:33,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:33,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +19: [2023-05-13 09:38:33,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +15: [2023-05-13 09:38:33,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +15: [2023-05-13 09:38:33,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +21: [2023-05-13 09:38:33,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +15: [2023-05-13 09:38:33,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:33,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:33,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:33,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:33,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:33,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:33,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +17: [2023-05-13 09:38:33,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:33,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +18: [2023-05-13 09:38:33,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +17: [2023-05-13 09:38:33,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:33,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +18: [2023-05-13 09:38:33,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +18: [2023-05-13 09:38:33,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:33,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:33,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:33,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:33,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:33,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:33,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:33,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +18: [2023-05-13 09:38:33,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:33,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:33,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:33,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +28: [2023-05-13 09:38:33,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:33,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:33,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +16: [2023-05-13 09:38:33,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:33,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +23: [2023-05-13 09:38:33,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +23: [2023-05-13 09:38:33,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +23: [2023-05-13 09:38:33,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +11: [2023-05-13 09:38:33,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +26: [2023-05-13 09:38:33,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +26: [2023-05-13 09:38:33,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:33,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:33,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:33,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:33,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +24: [2023-05-13 09:38:33,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +19: [2023-05-13 09:38:33,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +15: [2023-05-13 09:38:33,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +18: [2023-05-13 09:38:33,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +25: [2023-05-13 09:38:33,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +25: [2023-05-13 09:38:33,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +18: [2023-05-13 09:38:33,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +14: [2023-05-13 09:38:33,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +19: [2023-05-13 09:38:33,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +19: [2023-05-13 09:38:33,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +18: [2023-05-13 09:38:33,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:33,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:33,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +20: [2023-05-13 09:38:33,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +20: [2023-05-13 09:38:33,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +20: [2023-05-13 09:38:33,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +25: [2023-05-13 09:38:33,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:33,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:33,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:33,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:33,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:33,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:33,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:33,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +20: [2023-05-13 09:38:33,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:33,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:33,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +14: [2023-05-13 09:38:33,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:33,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:33,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:33,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:33,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:33,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:33,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:33,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:33,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +16: [2023-05-13 09:38:33,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +16: [2023-05-13 09:38:33,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +16: [2023-05-13 09:38:33,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +12: [2023-05-13 09:38:33,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:33,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:33,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:33,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:33,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:33,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +22: [2023-05-13 09:38:33,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +22: [2023-05-13 09:38:33,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +22: [2023-05-13 09:38:33,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:33,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:33,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +23: [2023-05-13 09:38:33,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:33,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:33,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:33,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +14: [2023-05-13 09:38:33,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +14: [2023-05-13 09:38:33,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +26: [2023-05-13 09:38:33,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:33,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:33,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:33,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:33,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:33,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +24: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:33,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:33,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:33,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:33,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:33,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +24: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:33,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:33,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:33,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:33,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:33,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:33,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:33,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:33,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:33,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:33,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:33,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:33,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:33,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:33,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:33,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +11: [2023-05-13 09:38:33,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:33,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +11: [2023-05-13 09:38:33,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:33,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +28: [2023-05-13 09:38:33,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:34,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:34,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +26: [2023-05-13 09:38:34,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:34,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:34,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +14: [2023-05-13 09:38:34,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:34,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:34,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:34,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +26: [2023-05-13 09:38:34,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:34,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:34,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:34,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +14: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:34,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:34,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:34,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +13: [2023-05-13 09:38:34,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +13: [2023-05-13 09:38:34,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:34,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:34,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:34,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:34,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:34,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:34,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:34,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:34,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:34,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:34,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:34,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +20: [2023-05-13 09:38:34,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:34,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:34,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +27: [2023-05-13 09:38:34,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +27: [2023-05-13 09:38:34,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +27: [2023-05-13 09:38:34,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:34,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +29: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +29: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +29: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:34,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:34,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +13: [2023-05-13 09:38:34,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +17: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +17: [2023-05-13 09:38:34,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:33,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:33,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:33,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:33,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:34,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:34,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:34,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:34,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:34,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:34,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:34,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:34,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:34,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:34,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:34,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +26: [2023-05-13 09:38:34,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:34,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:34,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:34,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:34,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:34,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:34,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:34,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:34,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +26: [2023-05-13 09:38:34,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:34,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:34,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +26: [2023-05-13 09:38:34,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +26: [2023-05-13 09:38:34,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:34,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:34,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:34,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:34,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:34,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:34,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +29: [2023-05-13 09:38:34,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:34,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:34,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +13: [2023-05-13 09:38:34,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +13: [2023-05-13 09:38:34,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +29: [2023-05-13 09:38:34,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:34,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:34,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:34,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:34,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:34,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:34,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:34,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +29: [2023-05-13 09:38:34,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +31: [2023-05-13 09:38:34,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:34,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:34,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:34,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:34,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:34,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:34,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:34,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:34,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +21: [2023-05-13 09:38:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +17: [2023-05-13 09:38:34,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +17: [2023-05-13 09:38:34,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:34,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:34,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:34,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:34,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:34,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:34,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +28: [2023-05-13 09:38:34,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +28: [2023-05-13 09:38:34,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:34,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +28: [2023-05-13 09:38:34,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +31: [2023-05-13 09:38:34,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:34,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:34,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:34,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:34,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:34,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:34,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:34,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:34,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:34,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:34,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +24: [2023-05-13 09:38:34,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +15: [2023-05-13 09:38:34,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... +31: [2023-05-13 09:38:34,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:34,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:34,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:34,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:34,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +19: [2023-05-13 09:38:34,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +19: [2023-05-13 09:38:34,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +19: [2023-05-13 09:38:34,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:34,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:34,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:34,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:34,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:34,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +21: [2023-05-13 09:38:34,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +19: [2023-05-13 09:38:34,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:34,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:34,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:34,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:34,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:34,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +17: [2023-05-13 09:38:34,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +15: [2023-05-13 09:38:34,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:34,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +21: [2023-05-13 09:38:34,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:34,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +21: [2023-05-13 09:38:34,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:34,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +17: [2023-05-13 09:38:34,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:34,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:34,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:34,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:34,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:34,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:34,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:34,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:34,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:34,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:34,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:34,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:34,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +27: [2023-05-13 09:38:34,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +15: [2023-05-13 09:38:34,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:34,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:34,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:34,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:34,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +27: [2023-05-13 09:38:34,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +27: [2023-05-13 09:38:34,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +27: [2023-05-13 09:38:34,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +25: [2023-05-13 09:38:34,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +21: [2023-05-13 09:38:34,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +23: [2023-05-13 09:38:34,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +23: [2023-05-13 09:38:34,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +23: [2023-05-13 09:38:34,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:34,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:34,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +23: [2023-05-13 09:38:34,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +21: [2023-05-13 09:38:34,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:34,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +21: [2023-05-13 09:38:34,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +21: [2023-05-13 09:38:34,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:34,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:34,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:34,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:34,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:34,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:34,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:34,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:34,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:34,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:34,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:34,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:34,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +18: [2023-05-13 09:38:34,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +18: [2023-05-13 09:38:34,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:34,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:34,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:34,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:34,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:34,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:34,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:34,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:34,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:34,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:34,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:34,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +14: [2023-05-13 09:38:34,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +14: [2023-05-13 09:38:34,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +14: [2023-05-13 09:38:34,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 6: [2023-05-13 09:38:34,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:34,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +18: [2023-05-13 09:38:34,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:34,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:34,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:34,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:34,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +14: [2023-05-13 09:38:34,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:34,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:34,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +14: [2023-05-13 09:38:34,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:34,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:34,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:34,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:34,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:34,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:34,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:34,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:34,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:34,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +23: [2023-05-13 09:38:34,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +23: [2023-05-13 09:38:34,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:34,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:34,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +23: [2023-05-13 09:38:34,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:34,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:34,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:34,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:34,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:34,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:34,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +14: [2023-05-13 09:38:34,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:34,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:34,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:34,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:34,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +23: [2023-05-13 09:38:34,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:34,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:34,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:34,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:34,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:34,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:34,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:34,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:34,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:34,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:34,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:34,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:34,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:34,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:34,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:34,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +25: [2023-05-13 09:38:34,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +25: [2023-05-13 09:38:34,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:34,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +24: [2023-05-13 09:38:34,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +24: [2023-05-13 09:38:34,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:34,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +25: [2023-05-13 09:38:34,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +25: [2023-05-13 09:38:34,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:34,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +28: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +28: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +26: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +26: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +26: [2023-05-13 09:38:34,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +28: [2023-05-13 09:38:34,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:34,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:34,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:34,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:34,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:34,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:34,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +22: [2023-05-13 09:38:34,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +22: [2023-05-13 09:38:34,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +22: [2023-05-13 09:38:34,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +13: [2023-05-13 09:38:34,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +13: [2023-05-13 09:38:34,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +13: [2023-05-13 09:38:34,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +13: [2023-05-13 09:38:34,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:34,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:34,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:34,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:34,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +13: [2023-05-13 09:38:34,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:34,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:34,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:34,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:34,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:34,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:34,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:34,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:34,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:34,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:34,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:34,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:34,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +10: [2023-05-13 09:38:34,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +10: [2023-05-13 09:38:34,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:34,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:34,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:34,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +20: [2023-05-13 09:38:34,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:34,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +31: [2023-05-13 09:38:34,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:34,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +31: [2023-05-13 09:38:34,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +13: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +13: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:34,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:34,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:34,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:34,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:34,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +19: [2023-05-13 09:38:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +19: [2023-05-13 09:38:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +12: [2023-05-13 09:38:34,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +22: [2023-05-13 09:38:34,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:34,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:34,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +24: [2023-05-13 09:38:34,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:34,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +17: [2023-05-13 09:38:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:34,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +21: [2023-05-13 09:38:34,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt... +24: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 6: [2023-05-13 09:38:34,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:34,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:34,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:34,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:34,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +27: [2023-05-13 09:38:34,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:34,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:34,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 6: [2023-05-13 09:38:34,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:34,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:34,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:34,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +19: [2023-05-13 09:38:34,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:34,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:34,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +31: [2023-05-13 09:38:34,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:34,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:34,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:34,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:34,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:34,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:34,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:34,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:34,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +13: [2023-05-13 09:38:34,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:34,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:34,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:34,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:34,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +12: [2023-05-13 09:38:34,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +12: [2023-05-13 09:38:34,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:34,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:34,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:34,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:34,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:34,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +24: [2023-05-13 09:38:34,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:34,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:34,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:34,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:34,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:34,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:34,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +21: [2023-05-13 09:38:34,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:34,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:34,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:34,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:34,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:34,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:34,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +23: [2023-05-13 09:38:34,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +23: [2023-05-13 09:38:34,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +23: [2023-05-13 09:38:34,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +23: [2023-05-13 09:38:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +31: [2023-05-13 09:38:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +31: [2023-05-13 09:38:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +31: [2023-05-13 09:38:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:34,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:34,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:34,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:34,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:34,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +21: [2023-05-13 09:38:34,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +27: [2023-05-13 09:38:34,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:34,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +18: [2023-05-13 09:38:34,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:34,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:34,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:34,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:34,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +18: [2023-05-13 09:38:34,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:34,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +18: [2023-05-13 09:38:34,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:34,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:34,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:34,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:34,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:34,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +23: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +23: [2023-05-13 09:38:34,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:34,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:34,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:34,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:34,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:34,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:34,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:34,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +31: [2023-05-13 09:38:34,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +18: [2023-05-13 09:38:34,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:34,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:34,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:34,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +31: [2023-05-13 09:38:34,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:34,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:34,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:34,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:34,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:34,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:34,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:34,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:34,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:34,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:34,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:34,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +20: [2023-05-13 09:38:34,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +20: [2023-05-13 09:38:34,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +20: [2023-05-13 09:38:34,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +20: [2023-05-13 09:38:34,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:34,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +30: [2023-05-13 09:38:34,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +30: [2023-05-13 09:38:34,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:34,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:34,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:34,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:34,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:34,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:34,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:34,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:34,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +20: [2023-05-13 09:38:34,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +21: [2023-05-13 09:38:34,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +30: [2023-05-13 09:38:34,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +21: [2023-05-13 09:38:34,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +30: [2023-05-13 09:38:34,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:34,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +23: [2023-05-13 09:38:34,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +23: [2023-05-13 09:38:34,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:34,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:34,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:34,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +21: [2023-05-13 09:38:34,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:34,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +20: [2023-05-13 09:38:34,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +20: [2023-05-13 09:38:34,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:34,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:34,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:34,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:34,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:34,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:34,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:34,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:34,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:34,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:34,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +16: [2023-05-13 09:38:34,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +16: [2023-05-13 09:38:34,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:34,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:34,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +23: [2023-05-13 09:38:34,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:34,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:34,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:34,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:34,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:34,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:34,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:34,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +21: [2023-05-13 09:38:34,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:34,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:34,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:34,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:34,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:34,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:34,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:34,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:34,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:34,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:34,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:34,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +28: [2023-05-13 09:38:34,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:34,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:34,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:34,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:34,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:34,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:34,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:34,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:34,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:34,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:34,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:34,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:34,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:34,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:34,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:34,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:34,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:34,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:34,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:34,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:34,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:34,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:34,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:34,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +24: [2023-05-13 09:38:34,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:34,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:34,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:34,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:34,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +13: [2023-05-13 09:38:34,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:34,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +13: [2023-05-13 09:38:34,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:34,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +12: [2023-05-13 09:38:34,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +12: [2023-05-13 09:38:34,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +12: [2023-05-13 09:38:34,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +22: [2023-05-13 09:38:34,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +22: [2023-05-13 09:38:34,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +22: [2023-05-13 09:38:34,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +22: [2023-05-13 09:38:34,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +12: [2023-05-13 09:38:34,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +12: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +24: [2023-05-13 09:38:34,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +24: [2023-05-13 09:38:34,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +10: [2023-05-13 09:38:34,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +10: [2023-05-13 09:38:34,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +10: [2023-05-13 09:38:34,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +10: [2023-05-13 09:38:34,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +12: [2023-05-13 09:38:34,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:34,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +13: [2023-05-13 09:38:34,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +13: [2023-05-13 09:38:34,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:34,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:34,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:34,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:34,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:34,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:34,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:34,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:34,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:34,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +28: [2023-05-13 09:38:34,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +28: [2023-05-13 09:38:34,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:34,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:34,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +18: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:34,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:34,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:34,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +18: [2023-05-13 09:38:34,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:34,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:34,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +21: [2023-05-13 09:38:34,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:34,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:34,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:34,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +21: [2023-05-13 09:38:34,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:34,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +12: [2023-05-13 09:38:34,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:34,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:34,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:34,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +29: [2023-05-13 09:38:34,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +13: [2023-05-13 09:38:34,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +13: [2023-05-13 09:38:34,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:34,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:34,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:34,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:34,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:34,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +12: [2023-05-13 09:38:34,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:34,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:34,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +10: [2023-05-13 09:38:34,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +10: [2023-05-13 09:38:34,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:34,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:34,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:34,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:34,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:34,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:34,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:34,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:34,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:34,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:34,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:34,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:34,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:34,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:34,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:34,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:34,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +15: [2023-05-13 09:38:34,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +15: [2023-05-13 09:38:34,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +12: [2023-05-13 09:38:34,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:34,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:34,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:34,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:34,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:34,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:34,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +14: [2023-05-13 09:38:34,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +14: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:34,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:34,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +29: [2023-05-13 09:38:34,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:34,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:34,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +15: [2023-05-13 09:38:35,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:35,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:35,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:35,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:35,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:35,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +29: [2023-05-13 09:38:35,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:35,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:35,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:35,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:35,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +23: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:35,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:35,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +23: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +23: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +20: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:35,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:35,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:35,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:35,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:35,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:35,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:35,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:35,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:35,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:35,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:35,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:35,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:35,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:35,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +16: [2023-05-13 09:38:35,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:35,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:35,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:35,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:35,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:35,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:35,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:35,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:35,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:35,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:35,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +23: [2023-05-13 09:38:35,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:35,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:35,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:35,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:35,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +23: [2023-05-13 09:38:35,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:35,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:35,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:35,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +16: [2023-05-13 09:38:35,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:35,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:35,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:35,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:35,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +17: [2023-05-13 09:38:35,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:35,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:35,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:35,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +14: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +14: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +14: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:35,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +20: [2023-05-13 09:38:35,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:35,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:35,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:35,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:35,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:35,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:35,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:35,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:35,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:35,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:35,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:35,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:35,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:35,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:35,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:35,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:35,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:35,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:35,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:35,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:35,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:35,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:35,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:35,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:35,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:35,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:35,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:35,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:35,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:35,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:35,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:35,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:35,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:35,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:35,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:35,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:35,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:35,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:35,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:35,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 5: [2023-05-13 09:38:35,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:35,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 5: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:35,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 7: [2023-05-13 09:38:35,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 7: [2023-05-13 09:38:35,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:35,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:35,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +25: [2023-05-13 09:38:35,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +25: [2023-05-13 09:38:35,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +25: [2023-05-13 09:38:35,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:35,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:35,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:35,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:35,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:35,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +30: [2023-05-13 09:38:35,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +30: [2023-05-13 09:38:35,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +30: [2023-05-13 09:38:35,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:35,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:35,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +14: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:35,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:35,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:35,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:35,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:35,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:35,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:35,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:35,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:35,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:35,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:35,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:35,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:35,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:35,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:35,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:35,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:35,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:35,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:35,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:35,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:35,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:35,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:35,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:35,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:35,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:35,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:35,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:35,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:35,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:35,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:35,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +25: [2023-05-13 09:38:35,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:35,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +23: [2023-05-13 09:38:35,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +23: [2023-05-13 09:38:35,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:35,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:35,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:35,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:35,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:35,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:35,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:35,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:35,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:35,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:35,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:35,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:35,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:35,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:35,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:35,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:35,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:35,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:35,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:35,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +28: [2023-05-13 09:38:35,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:35,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:35,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:35,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:35,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:35,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:35,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:35,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:35,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:35,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:35,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +25: [2023-05-13 09:38:35,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:35,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +25: [2023-05-13 09:38:35,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:35,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +18: [2023-05-13 09:38:35,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +18: [2023-05-13 09:38:35,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:35,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +25: [2023-05-13 09:38:35,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +18: [2023-05-13 09:38:35,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +24: [2023-05-13 09:38:35,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +25: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:35,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:35,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +26: [2023-05-13 09:38:35,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +26: [2023-05-13 09:38:35,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +26: [2023-05-13 09:38:35,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:35,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:35,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:35,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +11: [2023-05-13 09:38:35,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +11: [2023-05-13 09:38:35,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +11: [2023-05-13 09:38:35,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:35,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:35,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:35,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:35,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:35,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +28: [2023-05-13 09:38:35,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +28: [2023-05-13 09:38:35,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:35,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:35,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:35,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:35,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +28: [2023-05-13 09:38:35,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:35,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:35,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:35,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:35,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:35,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:35,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:35,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +19: [2023-05-13 09:38:35,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +19: [2023-05-13 09:38:35,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +19: [2023-05-13 09:38:35,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:35,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:35,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +20: [2023-05-13 09:38:35,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:35,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:35,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:35,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:35,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:35,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:35,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:35,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:35,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:35,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:35,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:35,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:35,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:35,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:35,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:35,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:35,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:35,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:35,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:35,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:35,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:35,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:35,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:35,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +12: [2023-05-13 09:38:35,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:35,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:35,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:35,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:35,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +20: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +13: [2023-05-13 09:38:35,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:35,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +22: [2023-05-13 09:38:35,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +22: [2023-05-13 09:38:35,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +22: [2023-05-13 09:38:35,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +20: [2023-05-13 09:38:35,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:35,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +20: [2023-05-13 09:38:35,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +20: [2023-05-13 09:38:35,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:35,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:35,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:35,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:35,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:35,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:35,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:35,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:35,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:35,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:35,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:35,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:35,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:35,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:35,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:35,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:35,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:35,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:35,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:35,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:35,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +16: [2023-05-13 09:38:35,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:35,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:35,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... + 7: [2023-05-13 09:38:35,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:35,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:35,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:35,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +13: [2023-05-13 09:38:35,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +16: [2023-05-13 09:38:35,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt... +13: [2023-05-13 09:38:35,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +13: [2023-05-13 09:38:35,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +13: [2023-05-13 09:38:35,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +13: [2023-05-13 09:38:35,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:35,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +23: [2023-05-13 09:38:35,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +23: [2023-05-13 09:38:35,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +23: [2023-05-13 09:38:35,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:35,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:35,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:35,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:35,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:35,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +23: [2023-05-13 09:38:35,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:35,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:35,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:35,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:35,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:35,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:35,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:35,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:35,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:35,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:35,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:35,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:35,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:35,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:35,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +11: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +15: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +15: [2023-05-13 09:38:35,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +15: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +30: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +15: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +30: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +30: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +30: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:35,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +14: [2023-05-13 09:38:35,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +14: [2023-05-13 09:38:35,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +24: [2023-05-13 09:38:35,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +27: [2023-05-13 09:38:35,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +27: [2023-05-13 09:38:35,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +27: [2023-05-13 09:38:35,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +24: [2023-05-13 09:38:35,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +24: [2023-05-13 09:38:35,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:35,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:35,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +25: [2023-05-13 09:38:35,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:35,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +25: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +11: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +27: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +14: [2023-05-13 09:38:35,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +31: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +28: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +28: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:35,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +28: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +20: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:35,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:35,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:35,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:35,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:35,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +29: [2023-05-13 09:38:35,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:35,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +22: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +22: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +30: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +29: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +29: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +26: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +20: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +20: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:35,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +11: [2023-05-13 09:38:35,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 7: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 7: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +11: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +31: [2023-05-13 09:38:35,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +30: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:35,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:35,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +31: [2023-05-13 09:38:35,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:35,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:35,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:35,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +30: [2023-05-13 09:38:35,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +27: [2023-05-13 09:38:35,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 5: [2023-05-13 09:38:35,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:35,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:35,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:35,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:35,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +25: [2023-05-13 09:38:35,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +30: [2023-05-13 09:38:35,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +11: [2023-05-13 09:38:35,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:35,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +20: [2023-05-13 09:38:35,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +20: [2023-05-13 09:38:35,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:35,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:35,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +20: [2023-05-13 09:38:35,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:35,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:35,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:35,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:35,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:35,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:35,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +25: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:35,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +22: [2023-05-13 09:38:35,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:35,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +29: [2023-05-13 09:38:35,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:35,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +16: [2023-05-13 09:38:35,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +25: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +26: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +24: [2023-05-13 09:38:35,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +20: [2023-05-13 09:38:35,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:35,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:35,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:35,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:35,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +10: [2023-05-13 09:38:35,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:35,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +20: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:35,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:35,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +28: [2023-05-13 09:38:35,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:35,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:35,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:35,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:35,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:35,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:35,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:35,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +26: [2023-05-13 09:38:35,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +26: [2023-05-13 09:38:35,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +26: [2023-05-13 09:38:35,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:35,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:35,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:35,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:35,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +16: [2023-05-13 09:38:35,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +16: [2023-05-13 09:38:35,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +16: [2023-05-13 09:38:35,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +16: [2023-05-13 09:38:35,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_14-model_01-model_states.pt. +16: [2023-05-13 09:38:35,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:35,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:35,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:35,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:35,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +16: [2023-05-13 09:38:35,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +16: [2023-05-13 09:38:35,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +16: [2023-05-13 09:38:35,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt... +13: [2023-05-13 09:38:35,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +31: [2023-05-13 09:38:35,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +31: [2023-05-13 09:38:35,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +31: [2023-05-13 09:38:35,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +31: [2023-05-13 09:38:35,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:35,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:35,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:35,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:35,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:35,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:35,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:35,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:35,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:35,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:35,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +22: [2023-05-13 09:38:35,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +22: [2023-05-13 09:38:35,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +22: [2023-05-13 09:38:35,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +22: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +12: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +12: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +12: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +12: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +27: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +27: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +27: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +27: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:35,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +19: [2023-05-13 09:38:35,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +15: [2023-05-13 09:38:35,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +15: [2023-05-13 09:38:35,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +15: [2023-05-13 09:38:35,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +19: [2023-05-13 09:38:35,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +19: [2023-05-13 09:38:35,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +19: [2023-05-13 09:38:35,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +15: [2023-05-13 09:38:35,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:35,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +30: [2023-05-13 09:38:35,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +30: [2023-05-13 09:38:35,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:35,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:35,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:35,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:35,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:35,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:35,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:35,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +23: [2023-05-13 09:38:35,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +23: [2023-05-13 09:38:35,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +27: [2023-05-13 09:38:35,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:35,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +25: [2023-05-13 09:38:35,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +25: [2023-05-13 09:38:35,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +25: [2023-05-13 09:38:35,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +25: [2023-05-13 09:38:35,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +15: [2023-05-13 09:38:35,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +15: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +15: [2023-05-13 09:38:35,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +15: [2023-05-13 09:38:35,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:35,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:35,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:35,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +23: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +25: [2023-05-13 09:38:35,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +16: [2023-05-13 09:38:35,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +16: [2023-05-13 09:38:35,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. +16: [2023-05-13 09:38:35,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +24: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +21: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +15: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +21: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:35,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:35,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +15: [2023-05-13 09:38:35,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:35,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:35,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:35,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:35,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:35,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:35,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:35,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:35,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:35,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:35,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +15: [2023-05-13 09:38:35,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:35,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:35,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +26: [2023-05-13 09:38:35,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:35,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:35,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:35,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:35,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:35,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:35,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +14: [2023-05-13 09:38:35,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +14: [2023-05-13 09:38:35,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +16: [2023-05-13 09:38:35,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +14: [2023-05-13 09:38:35,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +14: [2023-05-13 09:38:35,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +16: [2023-05-13 09:38:35,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +16: [2023-05-13 09:38:35,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +30: [2023-05-13 09:38:35,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:35,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:35,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:35,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:35,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:35,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:35,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +29: [2023-05-13 09:38:35,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +13: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +29: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +21: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:35,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +10: [2023-05-13 09:38:35,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:35,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +20: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +20: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +10: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +26: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +26: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:35,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +16: [2023-05-13 09:38:35,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:35,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:35,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:35,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:35,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:35,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:35,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:35,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:35,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:35,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:35,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:35,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:35,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:35,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:35,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:35,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:35,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:36,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:36,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +24: [2023-05-13 09:38:36,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +22: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +24: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:36,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:36,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:36,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +22: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +22: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +22: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:36,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:36,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:36,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:36,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:36,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:36,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 7: [2023-05-13 09:38:36,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:36,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:36,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:36,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:36,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:36,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:36,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:36,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:36,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:36,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:36,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:36,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:36,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:36,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:36,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:36,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:36,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:36,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:36,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:36,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:36,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:36,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +26: [2023-05-13 09:38:36,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:36,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +26: [2023-05-13 09:38:36,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:36,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:36,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:36,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:36,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:36,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:36,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:36,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:36,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:36,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:36,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:36,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:36,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:36,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:36,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:36,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:36,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:36,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:36,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:36,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:36,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:36,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:36,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:36,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:36,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:36,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +18: [2023-05-13 09:38:36,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 7: [2023-05-13 09:38:36,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:36,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:36,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:36,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:36,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:36,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:36,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +18: [2023-05-13 09:38:36,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:36,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:36,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:36,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:36,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:36,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +18: [2023-05-13 09:38:36,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +22: [2023-05-13 09:38:36,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:36,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:36,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:36,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:36,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:36,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:36,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:36,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:36,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +19: [2023-05-13 09:38:36,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:36,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:36,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:36,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:36,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:36,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:36,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +23: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +23: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +19: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +23: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +19: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +19: [2023-05-13 09:38:36,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:36,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:36,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:36,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:36,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +25: [2023-05-13 09:38:36,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +25: [2023-05-13 09:38:36,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:36,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:36,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:36,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:36,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:36,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:36,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:36,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:36,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:36,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:36,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +25: [2023-05-13 09:38:36,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:36,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +25: [2023-05-13 09:38:36,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:36,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:36,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:36,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:36,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:36,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:36,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:36,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +22: [2023-05-13 09:38:36,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:36,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:36,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:36,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:36,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:36,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:36,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:36,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:36,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:36,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:36,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:36,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:36,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:36,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:36,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:36,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:36,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:36,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:36,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:36,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:36,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:36,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:36,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:36,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:36,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:36,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:36,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:36,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:36,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:36,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:36,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:36,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +16: [2023-05-13 09:38:36,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +16: [2023-05-13 09:38:36,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_16-model_01-model_states.pt. +25: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:36,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:36,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:36,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:36,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:36,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:36,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:36,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:36,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:36,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:36,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:36,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:36,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:36,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:36,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:36,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:36,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:36,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:36,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +12: [2023-05-13 09:38:36,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +12: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:36,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:36,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:36,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:36,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:36,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:36,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:36,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +12: [2023-05-13 09:38:36,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:36,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:36,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:36,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:36,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt... +15: [2023-05-13 09:38:36,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:36,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:36,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:36,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:36,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:36,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:36,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:36,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:36,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:36,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:36,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +30: [2023-05-13 09:38:36,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +30: [2023-05-13 09:38:36,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +30: [2023-05-13 09:38:36,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:36,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:36,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:36,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:36,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:36,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:36,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:36,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:36,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:36,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:36,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:36,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:36,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:36,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +21: [2023-05-13 09:38:36,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +21: [2023-05-13 09:38:36,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +21: [2023-05-13 09:38:36,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:36,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:36,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:36,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:36,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:36,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:36,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +21: [2023-05-13 09:38:36,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:36,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:36,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:36,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:36,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:36,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:36,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:36,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:36,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:36,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:36,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:36,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:36,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:36,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:36,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:36,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 4: [2023-05-13 09:38:36,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:36,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:36,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:36,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:36,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:36,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:36,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:36,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:36,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:36,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:36,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:36,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:36,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:36,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:36,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:36,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:36,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:36,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:36,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:36,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +22: [2023-05-13 09:38:36,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:36,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:36,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:36,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 7: [2023-05-13 09:38:36,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 4: [2023-05-13 09:38:36,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 4: [2023-05-13 09:38:36,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 7: [2023-05-13 09:38:36,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 7: [2023-05-13 09:38:36,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 7: [2023-05-13 09:38:36,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 7: [2023-05-13 09:38:36,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 4: [2023-05-13 09:38:36,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:36,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:36,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:36,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +17: [2023-05-13 09:38:36,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +17: [2023-05-13 09:38:36,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:36,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:36,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +17: [2023-05-13 09:38:36,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:36,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:36,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:36,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:36,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:36,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:36,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:36,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:36,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +11: [2023-05-13 09:38:36,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +11: [2023-05-13 09:38:36,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:36,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:36,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:36,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:36,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +16: [2023-05-13 09:38:36,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:36,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:36,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +16: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +23: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +23: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +16: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +16: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +29: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +24: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:36,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +29: [2023-05-13 09:38:36,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:36,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +25: [2023-05-13 09:38:36,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +19: [2023-05-13 09:38:36,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +19: [2023-05-13 09:38:36,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:36,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:36,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +22: [2023-05-13 09:38:36,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +22: [2023-05-13 09:38:36,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +22: [2023-05-13 09:38:36,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:36,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:36,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:36,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:36,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:36,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +24: [2023-05-13 09:38:36,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +24: [2023-05-13 09:38:36,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:36,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:36,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:36,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:36,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:36,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:36,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:36,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +14: [2023-05-13 09:38:36,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +19: [2023-05-13 09:38:36,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:36,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:36,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:36,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +24: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +20: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +20: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +16: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:36,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:36,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:36,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +27: [2023-05-13 09:38:36,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:36,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:36,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +30: [2023-05-13 09:38:36,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:36,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:36,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +13: [2023-05-13 09:38:36,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +13: [2023-05-13 09:38:36,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +16: [2023-05-13 09:38:36,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +30: [2023-05-13 09:38:36,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +18: [2023-05-13 09:38:36,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +18: [2023-05-13 09:38:36,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:36,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:36,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:36,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:36,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +18: [2023-05-13 09:38:36,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:36,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +24: [2023-05-13 09:38:36,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:36,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:36,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:36,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:36,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:36,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +15: [2023-05-13 09:38:36,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +15: [2023-05-13 09:38:36,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +15: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +14: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +14: [2023-05-13 09:38:36,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +16: [2023-05-13 09:38:36,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:36,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:36,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:36,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +20: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:36,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:36,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +20: [2023-05-13 09:38:36,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:36,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:36,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:36,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +16: [2023-05-13 09:38:36,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +27: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +27: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:36,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:36,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:36,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 8: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +11: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +11: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:36,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +22: [2023-05-13 09:38:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:36,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:36,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 1: [2023-05-13 09:38:36,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 1: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 1: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 1: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +26: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:36,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:36,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:36,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +26: [2023-05-13 09:38:36,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:36,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:36,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:36,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:36,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:36,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +25: [2023-05-13 09:38:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:36,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +11: [2023-05-13 09:38:36,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:36,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:36,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:36,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:36,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +16: [2023-05-13 09:38:36,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +20: [2023-05-13 09:38:36,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +27: [2023-05-13 09:38:36,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +14: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +20: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +14: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +13: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +13: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +20: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +27: [2023-05-13 09:38:36,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +19: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +16: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +16: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +13: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +27: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +16: [2023-05-13 09:38:36,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +14: [2023-05-13 09:38:36,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +19: [2023-05-13 09:38:36,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +11: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +27: [2023-05-13 09:38:36,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +19: [2023-05-13 09:38:36,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +19: [2023-05-13 09:38:36,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +26: [2023-05-13 09:38:36,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +19: [2023-05-13 09:38:36,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +19: [2023-05-13 09:38:36,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 8: [2023-05-13 09:38:36,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +27: [2023-05-13 09:38:36,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:36,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +20: [2023-05-13 09:38:36,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +20: [2023-05-13 09:38:36,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:36,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:36,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +26: [2023-05-13 09:38:36,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:36,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:36,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:36,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:36,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:36,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:36,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:36,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt... + 8: [2023-05-13 09:38:36,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:36,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:36,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:36,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:36,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:36,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:36,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:36,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:36,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:36,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:36,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:36,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:36,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:36,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:36,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:36,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:36,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:36,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:36,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:36,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:36,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:36,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:36,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:36,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +10: [2023-05-13 09:38:36,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:36,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +31: [2023-05-13 09:38:36,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +31: [2023-05-13 09:38:36,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +31: [2023-05-13 09:38:36,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:36,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:36,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:36,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:36,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:36,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:36,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:36,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +30: [2023-05-13 09:38:36,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +30: [2023-05-13 09:38:36,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +30: [2023-05-13 09:38:36,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +26: [2023-05-13 09:38:36,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +10: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +10: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +10: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +31: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +31: [2023-05-13 09:38:36,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +26: [2023-05-13 09:38:36,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 4: [2023-05-13 09:38:36,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 4: [2023-05-13 09:38:36,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 4: [2023-05-13 09:38:36,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +26: [2023-05-13 09:38:36,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:36,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:36,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:36,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:36,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:36,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:36,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:36,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:36,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:36,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:36,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:36,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:36,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +21: [2023-05-13 09:38:36,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +21: [2023-05-13 09:38:36,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +10: [2023-05-13 09:38:36,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:36,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:36,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:36,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 4: [2023-05-13 09:38:36,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:36,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 4: [2023-05-13 09:38:36,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +10: [2023-05-13 09:38:36,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 4: [2023-05-13 09:38:36,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +10: [2023-05-13 09:38:36,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +10: [2023-05-13 09:38:36,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 4: [2023-05-13 09:38:36,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:36,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:36,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +29: [2023-05-13 09:38:36,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +29: [2023-05-13 09:38:36,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:36,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 4: [2023-05-13 09:38:36,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:36,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:36,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:36,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:36,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:36,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:36,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:36,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:36,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:36,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 7: [2023-05-13 09:38:36,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 7: [2023-05-13 09:38:36,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:36,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:36,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:36,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:36,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:36,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 8: [2023-05-13 09:38:36,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:36,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +17: [2023-05-13 09:38:36,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +17: [2023-05-13 09:38:36,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +17: [2023-05-13 09:38:36,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:36,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:36,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:36,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:36,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +19: [2023-05-13 09:38:36,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +19: [2023-05-13 09:38:36,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +19: [2023-05-13 09:38:36,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +19: [2023-05-13 09:38:36,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 8: [2023-05-13 09:38:36,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:36,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +27: [2023-05-13 09:38:36,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +27: [2023-05-13 09:38:36,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +17: [2023-05-13 09:38:36,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +18: [2023-05-13 09:38:36,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +18: [2023-05-13 09:38:36,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:36,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:36,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +17: [2023-05-13 09:38:36,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:36,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:36,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 8: [2023-05-13 09:38:36,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:36,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:36,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:36,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:36,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:36,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:36,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:36,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +15: [2023-05-13 09:38:36,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:36,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:36,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:36,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:36,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +18: [2023-05-13 09:38:36,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:36,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +15: [2023-05-13 09:38:36,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:36,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +20: [2023-05-13 09:38:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +20: [2023-05-13 09:38:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +18: [2023-05-13 09:38:36,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +20: [2023-05-13 09:38:36,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +17: [2023-05-13 09:38:36,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:36,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +12: [2023-05-13 09:38:36,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +18: [2023-05-13 09:38:36,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +17: [2023-05-13 09:38:36,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:36,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +17: [2023-05-13 09:38:36,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +17: [2023-05-13 09:38:36,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +17: [2023-05-13 09:38:36,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:36,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:36,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:36,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:36,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:36,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +22: [2023-05-13 09:38:36,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +22: [2023-05-13 09:38:36,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +22: [2023-05-13 09:38:36,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:36,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +11: [2023-05-13 09:38:36,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +11: [2023-05-13 09:38:36,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:36,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:36,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:36,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +19: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +23: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +27: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +12: [2023-05-13 09:38:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +19: [2023-05-13 09:38:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:36,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:36,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:36,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +27: [2023-05-13 09:38:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +27: [2023-05-13 09:38:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:36,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +17: [2023-05-13 09:38:36,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:36,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:36,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:36,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:36,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:36,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:36,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:36,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:36,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:36,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +27: [2023-05-13 09:38:36,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:36,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:36,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +17: [2023-05-13 09:38:36,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:36,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:36,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:36,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:36,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:36,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:36,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:36,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:36,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:36,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:36,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:36,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +28: [2023-05-13 09:38:36,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:36,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:36,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:36,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:36,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:36,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:36,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:36,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +26: [2023-05-13 09:38:36,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +26: [2023-05-13 09:38:36,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:36,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:36,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:36,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +12: [2023-05-13 09:38:36,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:36,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +12: [2023-05-13 09:38:36,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:36,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:36,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:36,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:36,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +11: [2023-05-13 09:38:36,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:36,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:36,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +11: [2023-05-13 09:38:36,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +16: [2023-05-13 09:38:36,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:36,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:36,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +11: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +24: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +24: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +11: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +11: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +24: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:36,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:36,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:36,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:36,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:36,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:36,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:36,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:36,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:36,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:36,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:36,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:36,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:36,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:36,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:36,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:36,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:36,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:36,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +21: [2023-05-13 09:38:36,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:36,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:36,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:36,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:36,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +20: [2023-05-13 09:38:36,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:36,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:36,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:36,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:36,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:36,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:36,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:36,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:36,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:36,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:36,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:36,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +20: [2023-05-13 09:38:36,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:36,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:36,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:36,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:36,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:36,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:36,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:36,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:36,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:36,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:36,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:36,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:36,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:36,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +28: [2023-05-13 09:38:36,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:36,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:36,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:36,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:36,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +24: [2023-05-13 09:38:36,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:36,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:36,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:36,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +13: [2023-05-13 09:38:36,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:36,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +13: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +13: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 1: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 1: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 1: [2023-05-13 09:38:36,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 1: [2023-05-13 09:38:36,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:36,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +16: [2023-05-13 09:38:36,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +16: [2023-05-13 09:38:36,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. +16: [2023-05-13 09:38:36,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_18-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:36,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:36,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:36,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:36,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:36,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:36,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:36,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:36,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:36,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:36,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:36,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:36,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +24: [2023-05-13 09:38:36,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:36,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:37,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:37,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:37,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:37,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +10: [2023-05-13 09:38:37,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:37,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:37,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:37,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:37,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:37,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:37,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:37,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:37,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:37,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:37,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:37,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:37,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:37,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +13: [2023-05-13 09:38:37,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:37,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:37,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:37,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:37,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:37,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:37,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:36,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:37,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:37,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:37,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:37,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +15: [2023-05-13 09:38:37,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:37,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:37,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:37,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:37,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:37,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:37,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:37,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:37,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +16: [2023-05-13 09:38:37,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:37,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:37,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:37,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:37,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +10: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +10: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +10: [2023-05-13 09:38:37,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:37,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:37,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +21: [2023-05-13 09:38:37,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:37,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +21: [2023-05-13 09:38:37,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +27: [2023-05-13 09:38:37,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:37,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:37,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:37,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:37,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:37,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:37,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:37,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +23: [2023-05-13 09:38:37,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +23: [2023-05-13 09:38:37,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:37,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... +23: [2023-05-13 09:38:37,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +26: [2023-05-13 09:38:37,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +26: [2023-05-13 09:38:37,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +12: [2023-05-13 09:38:37,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +12: [2023-05-13 09:38:37,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +12: [2023-05-13 09:38:37,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:37,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:37,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:37,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:37,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:37,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +25: [2023-05-13 09:38:37,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:37,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:37,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:37,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:37,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +15: [2023-05-13 09:38:37,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:37,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:37,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:37,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:37,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:37,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:37,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +18: [2023-05-13 09:38:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +23: [2023-05-13 09:38:37,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +18: [2023-05-13 09:38:37,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +19: [2023-05-13 09:38:37,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +19: [2023-05-13 09:38:37,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +19: [2023-05-13 09:38:37,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:37,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:37,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:37,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:37,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:37,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +18: [2023-05-13 09:38:37,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:37,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:37,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:37,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +15: [2023-05-13 09:38:37,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +15: [2023-05-13 09:38:37,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:37,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:37,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:37,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:37,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:37,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:37,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:37,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:37,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +18: [2023-05-13 09:38:37,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +18: [2023-05-13 09:38:37,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:37,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:37,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:37,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:37,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:37,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:37,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:37,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:37,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:37,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:37,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:37,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +30: [2023-05-13 09:38:37,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +30: [2023-05-13 09:38:37,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +30: [2023-05-13 09:38:37,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:37,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:37,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +14: [2023-05-13 09:38:37,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:37,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +21: [2023-05-13 09:38:37,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:37,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:37,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:37,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +17: [2023-05-13 09:38:37,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:37,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:37,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +18: [2023-05-13 09:38:37,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:37,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:37,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:37,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:37,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +14: [2023-05-13 09:38:37,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:37,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +12: [2023-05-13 09:38:37,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:37,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:37,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +12: [2023-05-13 09:38:37,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +12: [2023-05-13 09:38:37,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:37,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +12: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:37,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:37,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:37,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:37,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:37,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:37,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:37,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:37,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:37,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:37,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:37,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +17: [2023-05-13 09:38:37,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +25: [2023-05-13 09:38:37,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +20: [2023-05-13 09:38:37,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +19: [2023-05-13 09:38:37,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +17: [2023-05-13 09:38:37,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +19: [2023-05-13 09:38:37,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:37,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:37,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:37,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +25: [2023-05-13 09:38:37,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:37,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:37,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +21: [2023-05-13 09:38:37,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:37,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:37,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:37,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:37,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:37,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:37,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:37,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:37,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:37,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:37,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:37,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:37,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +25: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:37,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +10: [2023-05-13 09:38:37,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:37,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:37,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:37,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:37,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:37,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:37,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:37,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +27: [2023-05-13 09:38:37,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +27: [2023-05-13 09:38:37,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:37,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +16: [2023-05-13 09:38:37,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +16: [2023-05-13 09:38:37,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:37,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:37,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:37,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:37,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:37,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +24: [2023-05-13 09:38:37,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +24: [2023-05-13 09:38:37,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +24: [2023-05-13 09:38:37,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +19: [2023-05-13 09:38:37,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +30: [2023-05-13 09:38:37,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +16: [2023-05-13 09:38:37,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:37,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:37,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:37,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:37,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:37,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:37,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +27: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:37,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:37,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +14: [2023-05-13 09:38:37,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:37,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:37,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:37,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:37,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:37,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:37,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:37,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:37,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:37,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:37,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +26: [2023-05-13 09:38:37,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:37,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:37,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:37,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:37,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:37,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:37,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:37,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:37,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:37,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:37,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:37,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:37,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:37,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:37,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:37,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:37,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:37,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:37,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:37,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +10: [2023-05-13 09:38:37,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:37,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:37,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:37,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:37,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:37,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:37,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:37,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:37,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:37,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:37,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +14: [2023-05-13 09:38:37,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:37,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:37,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:37,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:37,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:37,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:37,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:37,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:37,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:37,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:37,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:37,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:37,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:37,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:37,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:37,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:37,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:37,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:37,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:37,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:37,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:37,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:37,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:37,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:37,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:37,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:37,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:37,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:37,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:37,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:37,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:37,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:37,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:37,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +26: [2023-05-13 09:38:37,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:37,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:37,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:37,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:37,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:37,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:37,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:37,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +26: [2023-05-13 09:38:37,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:37,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:37,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:37,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:37,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:37,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:37,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:37,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:37,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:37,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:37,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:37,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:37,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +23: [2023-05-13 09:38:37,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +23: [2023-05-13 09:38:37,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:37,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:37,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:37,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:37,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:37,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:37,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:37,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:37,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:37,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:37,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:37,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:37,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:37,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:37,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:37,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:37,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:37,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... +15: [2023-05-13 09:38:37,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:37,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:37,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:37,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:37,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:37,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:37,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:37,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:37,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:37,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:37,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:37,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:37,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:37,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:37,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:37,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:37,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:37,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:37,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:37,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:37,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:37,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:37,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:37,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:37,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:37,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:37,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:37,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:37,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:37,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:37,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:37,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:37,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:37,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:37,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:37,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:37,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:37,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:37,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:37,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:37,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:37,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:37,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:37,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:37,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:37,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:37,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:37,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:37,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:37,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:37,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:37,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:37,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:37,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:37,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +21: [2023-05-13 09:38:37,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +21: [2023-05-13 09:38:37,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +21: [2023-05-13 09:38:37,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +29: [2023-05-13 09:38:37,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:37,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:37,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:37,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:37,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:37,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:37,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:37,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:37,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:37,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +23: [2023-05-13 09:38:37,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +18: [2023-05-13 09:38:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +14: [2023-05-13 09:38:37,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:37,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:37,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +15: [2023-05-13 09:38:37,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:37,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +15: [2023-05-13 09:38:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:37,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:37,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:37,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:37,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:37,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:37,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:37,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:37,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +10: [2023-05-13 09:38:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +10: [2023-05-13 09:38:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +23: [2023-05-13 09:38:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +21: [2023-05-13 09:38:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +18: [2023-05-13 09:38:37,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:37,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +17: [2023-05-13 09:38:37,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +15: [2023-05-13 09:38:37,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:37,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:37,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:37,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:37,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:37,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +19: [2023-05-13 09:38:37,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:37,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:37,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +30: [2023-05-13 09:38:37,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:37,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:37,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:37,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:37,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:37,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:37,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:37,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +18: [2023-05-13 09:38:37,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:37,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +19: [2023-05-13 09:38:37,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:37,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +18: [2023-05-13 09:38:37,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:37,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:37,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:37,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:37,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +18: [2023-05-13 09:38:37,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:37,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:37,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:37,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:37,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:37,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +25: [2023-05-13 09:38:37,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:37,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:37,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:37,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:37,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:37,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:37,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:37,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +25: [2023-05-13 09:38:37,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:37,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:37,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:37,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:37,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:37,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:37,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:37,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:37,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:37,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:37,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:37,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:37,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:37,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +28: [2023-05-13 09:38:37,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:37,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:37,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:37,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +28: [2023-05-13 09:38:37,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +22: [2023-05-13 09:38:37,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +22: [2023-05-13 09:38:37,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +22: [2023-05-13 09:38:37,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +22: [2023-05-13 09:38:37,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:37,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +22: [2023-05-13 09:38:37,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +28: [2023-05-13 09:38:37,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:37,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:37,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +19: [2023-05-13 09:38:37,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:37,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:37,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:37,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +31: [2023-05-13 09:38:37,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +31: [2023-05-13 09:38:37,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +31: [2023-05-13 09:38:37,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +31: [2023-05-13 09:38:37,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:37,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:37,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +29: [2023-05-13 09:38:37,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +29: [2023-05-13 09:38:37,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +29: [2023-05-13 09:38:37,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:37,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +29: [2023-05-13 09:38:37,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:37,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +24: [2023-05-13 09:38:37,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +24: [2023-05-13 09:38:37,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:37,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:37,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:37,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:37,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +24: [2023-05-13 09:38:37,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:37,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:37,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:37,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:37,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:37,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:37,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:37,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:37,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:37,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +27: [2023-05-13 09:38:37,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +27: [2023-05-13 09:38:37,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +27: [2023-05-13 09:38:37,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:37,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +14: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +14: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:37,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +26: [2023-05-13 09:38:37,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +29: [2023-05-13 09:38:37,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:37,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +26: [2023-05-13 09:38:37,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +29: [2023-05-13 09:38:37,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +16: [2023-05-13 09:38:37,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:37,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:37,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:37,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:37,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:37,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:37,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:37,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +23: [2023-05-13 09:38:37,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +23: [2023-05-13 09:38:37,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +23: [2023-05-13 09:38:37,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +29: [2023-05-13 09:38:37,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:37,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +19: [2023-05-13 09:38:37,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +16: [2023-05-13 09:38:37,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +16: [2023-05-13 09:38:37,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:37,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +16: [2023-05-13 09:38:37,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:37,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:37,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +19: [2023-05-13 09:38:37,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:37,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:37,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +29: [2023-05-13 09:38:37,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +29: [2023-05-13 09:38:37,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:37,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:37,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +29: [2023-05-13 09:38:37,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:37,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:37,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:37,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:37,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:37,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:37,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:37,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:37,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +14: [2023-05-13 09:38:37,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:37,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:37,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:37,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:37,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:37,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +14: [2023-05-13 09:38:37,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:37,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +14: [2023-05-13 09:38:37,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:37,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:37,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +14: [2023-05-13 09:38:37,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:37,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:37,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:37,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:37,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +11: [2023-05-13 09:38:37,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +11: [2023-05-13 09:38:37,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +11: [2023-05-13 09:38:37,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +10: [2023-05-13 09:38:37,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:37,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:37,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:37,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:37,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:37,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +10: [2023-05-13 09:38:37,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +10: [2023-05-13 09:38:37,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_20-model_01-model_states.pt. +10: [2023-05-13 09:38:37,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:37,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:37,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:37,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +11: [2023-05-13 09:38:37,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:37,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:37,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:37,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:37,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:37,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:37,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:37,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:37,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +11: [2023-05-13 09:38:37,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +11: [2023-05-13 09:38:37,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:37,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:37,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:37,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:37,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:37,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:37,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:37,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:37,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:37,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:37,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:37,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:37,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:37,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:37,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:37,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:37,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:37,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:38,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:38,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:38,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:38,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:38,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +18: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +18: [2023-05-13 09:38:38,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +18: [2023-05-13 09:38:38,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:38,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:38,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:38,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:38,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:38,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:38,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +21: [2023-05-13 09:38:38,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +21: [2023-05-13 09:38:38,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:38,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:38,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:38,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:38,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:38,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:38,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:38,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:38,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:38,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:38,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:38,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:38,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +21: [2023-05-13 09:38:38,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +21: [2023-05-13 09:38:38,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:38,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:38,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:38,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:38,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:38,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:38,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:38,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:38,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:38,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:38,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:38,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:38,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:38,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:38,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:38,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +10: [2023-05-13 09:38:38,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +10: [2023-05-13 09:38:38,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +13: [2023-05-13 09:38:38,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:38,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +10: [2023-05-13 09:38:38,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:38,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +13: [2023-05-13 09:38:38,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +10: [2023-05-13 09:38:38,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:38,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:38,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:38,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:38,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:38,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:38,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +15: [2023-05-13 09:38:38,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:38,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:38,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:38,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:38,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +15: [2023-05-13 09:38:38,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +15: [2023-05-13 09:38:38,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +15: [2023-05-13 09:38:38,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:38,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:38,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:38,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:38,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:38,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:38,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:38,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:38,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:38,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:38,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:38,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:38,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:38,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:38,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:38,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:38,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:38,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:38,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:38,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +12: [2023-05-13 09:38:38,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:38,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +10: [2023-05-13 09:38:38,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:38,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +15: [2023-05-13 09:38:38,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +25: [2023-05-13 09:38:38,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:38,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:38,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:38,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:38,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:38,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +10: [2023-05-13 09:38:38,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:38,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:38,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:38,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:38,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:38,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:38,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:38,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:38,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:38,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:38,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:38,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:38,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:38,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:38,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:38,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:38,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:38,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:38,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:38,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:38,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:38,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:38,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +29: [2023-05-13 09:38:38,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:38,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:38,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:38,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:38,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:38,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:38,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:38,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +13: [2023-05-13 09:38:38,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +13: [2023-05-13 09:38:38,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +25: [2023-05-13 09:38:38,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:38,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +26: [2023-05-13 09:38:38,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +30: [2023-05-13 09:38:38,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +30: [2023-05-13 09:38:38,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:38,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +15: [2023-05-13 09:38:38,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:38,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:38,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:38,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:38,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:38,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:38,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +12: [2023-05-13 09:38:38,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:38,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +26: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:38,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +26: [2023-05-13 09:38:38,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:38,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:38,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:38,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:38,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:38,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +12: [2023-05-13 09:38:38,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:38,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:38,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:38,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:38,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:38,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:38,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +24: [2023-05-13 09:38:38,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +18: [2023-05-13 09:38:38,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:38,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:38,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:38,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +18: [2023-05-13 09:38:38,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +21: [2023-05-13 09:38:38,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:38,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:38,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:38,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +30: [2023-05-13 09:38:38,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +18: [2023-05-13 09:38:38,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:38,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +28: [2023-05-13 09:38:38,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:38,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:38,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +31: [2023-05-13 09:38:38,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:38,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:38,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:38,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:38,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:38,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:38,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:38,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:38,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:38,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +17: [2023-05-13 09:38:38,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +30: [2023-05-13 09:38:38,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:38,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:38,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +13: [2023-05-13 09:38:38,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:38,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:38,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:38,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:38,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:38,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:38,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +24: [2023-05-13 09:38:38,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +28: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +28: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:38,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:38,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:38,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:38,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:38,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:38,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:38,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +29: [2023-05-13 09:38:38,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +18: [2023-05-13 09:38:38,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:38,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +14: [2023-05-13 09:38:38,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +14: [2023-05-13 09:38:38,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +14: [2023-05-13 09:38:38,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:38,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:38,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:38,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:38,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:38,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +27: [2023-05-13 09:38:38,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +27: [2023-05-13 09:38:38,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:38,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:38,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:38,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:38,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:38,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:38,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:38,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:38,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:38,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:38,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:38,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +18: [2023-05-13 09:38:38,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:38,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:38,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +22: [2023-05-13 09:38:38,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:38,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:38,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +22: [2023-05-13 09:38:38,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:38,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:38,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:38,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:38,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:38,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:38,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +27: [2023-05-13 09:38:38,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +14: [2023-05-13 09:38:38,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:38,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:38,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:38,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:38,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:38,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:38,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:38,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +25: [2023-05-13 09:38:38,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +14: [2023-05-13 09:38:38,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:38,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:38,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:38,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +17: [2023-05-13 09:38:38,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +17: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:38,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:38,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:38,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:38,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:38,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:38,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:38,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:38,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +28: [2023-05-13 09:38:38,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:38,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:38,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:38,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +27: [2023-05-13 09:38:38,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +25: [2023-05-13 09:38:38,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:38,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +23: [2023-05-13 09:38:38,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:38,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:38,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:38,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +25: [2023-05-13 09:38:38,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:38,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:38,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:38,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:38,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:38,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:38,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:38,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:38,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:38,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:38,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:38,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:38,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:38,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:38,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +22: [2023-05-13 09:38:38,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +22: [2023-05-13 09:38:38,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:38,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:38,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:38,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:38,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:38,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:38,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:38,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:38,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:38,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:38,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:38,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:38,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:38,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:38,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:38,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:38,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:38,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:38,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:38,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:38,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:38,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:38,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:38,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:38,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:38,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +19: [2023-05-13 09:38:38,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:38,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:38,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:38,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:38,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:38,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:38,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:38,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:38,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:38,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +25: [2023-05-13 09:38:38,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +25: [2023-05-13 09:38:38,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:38,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:38,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +20: [2023-05-13 09:38:38,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:38,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:38,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:38,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:38,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:38,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:38,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:38,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +24: [2023-05-13 09:38:38,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:38,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:38,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +10: [2023-05-13 09:38:38,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:38,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:38,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:38,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:38,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:38,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +10: [2023-05-13 09:38:38,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:38,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:38,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:38,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:38,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +31: [2023-05-13 09:38:38,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:38,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +22: [2023-05-13 09:38:38,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +31: [2023-05-13 09:38:38,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:38,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:38,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:38,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:38,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:38,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:38,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:38,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:38,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:38,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:38,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:38,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:38,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +10: [2023-05-13 09:38:38,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +10: [2023-05-13 09:38:38,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:38,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:38,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +24: [2023-05-13 09:38:38,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:38,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +10: [2023-05-13 09:38:38,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 1: [2023-05-13 09:38:38,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +21: [2023-05-13 09:38:38,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 1: [2023-05-13 09:38:38,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +21: [2023-05-13 09:38:38,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:38,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:38,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:38,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:38,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:38,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +26: [2023-05-13 09:38:38,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +26: [2023-05-13 09:38:38,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +26: [2023-05-13 09:38:38,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 1: [2023-05-13 09:38:38,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:38,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:38,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:38,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:38,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +31: [2023-05-13 09:38:38,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +31: [2023-05-13 09:38:38,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +20: [2023-05-13 09:38:38,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +20: [2023-05-13 09:38:38,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +20: [2023-05-13 09:38:38,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:38,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:38,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:38,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +18: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +18: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:38,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:38,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:38,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:38,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:38,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:38,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:38,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:38,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:38,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:38,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:38,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:38,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:38,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:38,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:38,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:38,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:38,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:38,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:38,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:38,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:38,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +18: [2023-05-13 09:38:38,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:38,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +18: [2023-05-13 09:38:38,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:38,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:38,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:38,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +12: [2023-05-13 09:38:38,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:38,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:38,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:38,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:38,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:38,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:38,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +31: [2023-05-13 09:38:38,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:38,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:38,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:38,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:38,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:38,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:38,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:38,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:38,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:38,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +18: [2023-05-13 09:38:38,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:38,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:38,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:38,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:38,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:38,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:38,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:38,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:38,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:38,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:38,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:38,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:38,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:38,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:38,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:38,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +18: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +18: [2023-05-13 09:38:38,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:38,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +24: [2023-05-13 09:38:38,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:38,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +15: [2023-05-13 09:38:38,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +15: [2023-05-13 09:38:38,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:38,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:38,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +15: [2023-05-13 09:38:38,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:38,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +30: [2023-05-13 09:38:38,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +30: [2023-05-13 09:38:38,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +30: [2023-05-13 09:38:38,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +30: [2023-05-13 09:38:38,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:38,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:38,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:38,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:38,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:38,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:38,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:38,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:38,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:38,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +12: [2023-05-13 09:38:38,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:38,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:38,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:38,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:38,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:38,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:38,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:38,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:38,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:38,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:38,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +24: [2023-05-13 09:38:38,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:38,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:38,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:38,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:38,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:38,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:38,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +19: [2023-05-13 09:38:38,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:38,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:38,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:38,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:38,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:38,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +19: [2023-05-13 09:38:38,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +24: [2023-05-13 09:38:38,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:38,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:38,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:38,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +24: [2023-05-13 09:38:38,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:38,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +11: [2023-05-13 09:38:38,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:38,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:38,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:38,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +28: [2023-05-13 09:38:38,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +28: [2023-05-13 09:38:38,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +28: [2023-05-13 09:38:38,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:38,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:38,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:38,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:38,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:38,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +23: [2023-05-13 09:38:38,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +23: [2023-05-13 09:38:38,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +23: [2023-05-13 09:38:38,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +29: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +29: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +29: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +29: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:38,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:38,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:38,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:38,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +28: [2023-05-13 09:38:38,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:38,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:38,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +21: [2023-05-13 09:38:38,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:38,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +21: [2023-05-13 09:38:38,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +21: [2023-05-13 09:38:38,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:38,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:38,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:38,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:38,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:38,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:38,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:38,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:38,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:38,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:38,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:38,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:38,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:38,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:38,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:38,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:38,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:38,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:38,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:38,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +14: [2023-05-13 09:38:38,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +14: [2023-05-13 09:38:38,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:38,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:38,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:38,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:38,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:38,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +25: [2023-05-13 09:38:38,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +25: [2023-05-13 09:38:38,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +25: [2023-05-13 09:38:38,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +23: [2023-05-13 09:38:38,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:38,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:38,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +17: [2023-05-13 09:38:38,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +17: [2023-05-13 09:38:38,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:38,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:38,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:38,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:38,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:38,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:38,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:38,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:38,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +22: [2023-05-13 09:38:38,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +29: [2023-05-13 09:38:38,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +10: [2023-05-13 09:38:38,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:38,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:38,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:38,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:38,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:38,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:38,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:38,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:38,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:38,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:38,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:38,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:38,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:38,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +27: [2023-05-13 09:38:38,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:38,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:38,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:38,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:38,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +27: [2023-05-13 09:38:38,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:38,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:38,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:38,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:38,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:38,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:38,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:38,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:38,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:38,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:38,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:38,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:38,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:38,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:38,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:38,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:38,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:38,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:38,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:38,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:38,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:38,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:38,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:38,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:38,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:38,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:38,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:38,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +27: [2023-05-13 09:38:38,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +27: [2023-05-13 09:38:38,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +27: [2023-05-13 09:38:38,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +14: [2023-05-13 09:38:38,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:38,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:38,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +10: [2023-05-13 09:38:38,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:38,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:38,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:38,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:38,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +31: [2023-05-13 09:38:38,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +31: [2023-05-13 09:38:38,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +31: [2023-05-13 09:38:38,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:38,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:38,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:38,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:38,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:38,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:38,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:38,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:38,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:38,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +27: [2023-05-13 09:38:38,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:38,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:38,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +17: [2023-05-13 09:38:38,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +17: [2023-05-13 09:38:38,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:38,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +24: [2023-05-13 09:38:38,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +17: [2023-05-13 09:38:38,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +24: [2023-05-13 09:38:38,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +17: [2023-05-13 09:38:38,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +10: [2023-05-13 09:38:38,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:38,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:38,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:38,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:38,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:38,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:38,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:38,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:38,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:38,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:38,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:38,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:38,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:38,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:38,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:38,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:38,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:38,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:38,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:38,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:38,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:38,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 1: [2023-05-13 09:38:38,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:38,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +27: [2023-05-13 09:38:38,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:38,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +31: [2023-05-13 09:38:38,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:38,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +25: [2023-05-13 09:38:38,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:38,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:38,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:38,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:38,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:38,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +24: [2023-05-13 09:38:38,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +26: [2023-05-13 09:38:38,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +26: [2023-05-13 09:38:38,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:38,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:38,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:38,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +24: [2023-05-13 09:38:38,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:38,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:38,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:38,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:38,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:38,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:38,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:38,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:38,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:38,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:38,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:38,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:38,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:38,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:38,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:38,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:38,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:38,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:38,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:38,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:38,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:38,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:38,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:38,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:38,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +13: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +12: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +16: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:38,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +12: [2023-05-13 09:38:38,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +12: [2023-05-13 09:38:38,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:38,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:38,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:38,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:38,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:38,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:38,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:38,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:38,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:38,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:38,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:38,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:38,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:38,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:38,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:38,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:38,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:38,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +10: [2023-05-13 09:38:38,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +10: [2023-05-13 09:38:38,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +10: [2023-05-13 09:38:38,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:38,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:38,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:38,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:38,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:38,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:38,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:38,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:38,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:38,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:38,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:38,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:38,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:38,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:38,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:38,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:38,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:38,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:38,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:38,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:38,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:38,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:38,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:38,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:38,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:38,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:38,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:38,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:38,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:38,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:38,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:38,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +12: [2023-05-13 09:38:38,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:38,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:38,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:38,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:38,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:38,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +30: [2023-05-13 09:38:38,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:38,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:38,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:38,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:38,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:38,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:38,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +21: [2023-05-13 09:38:38,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:38,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:38,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:38,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:38,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:38,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:38,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:38,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:38,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:38,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:38,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +11: [2023-05-13 09:38:38,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +11: [2023-05-13 09:38:38,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +11: [2023-05-13 09:38:38,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt... +21: [2023-05-13 09:38:38,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:39,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:39,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:39,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:39,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:39,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:39,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +21: [2023-05-13 09:38:39,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:39,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:39,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:39,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:39,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:39,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:39,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:39,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:39,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:39,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:39,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:39,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +10: [2023-05-13 09:38:39,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:39,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:39,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:39,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:39,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:39,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:39,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:39,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:39,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:39,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:39,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:39,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:39,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:39,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:39,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:39,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:39,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:39,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:39,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:39,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:39,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:39,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +18: [2023-05-13 09:38:39,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:39,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:39,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +18: [2023-05-13 09:38:39,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +18: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +18: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:39,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:39,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:39,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:39,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:39,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +23: [2023-05-13 09:38:39,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:39,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:39,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:39,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:39,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:39,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:39,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:39,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +24: [2023-05-13 09:38:39,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:39,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:39,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:39,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:39,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:39,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:39,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:39,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:39,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:39,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:39,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:39,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:39,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:39,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:39,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:39,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:39,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:39,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:39,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:39,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:39,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:39,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:39,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:39,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:39,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:39,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:39,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:39,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +31: [2023-05-13 09:38:39,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:39,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:39,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:39,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:39,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:39,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:39,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:39,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:39,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:39,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:39,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:39,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +28: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:39,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:39,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:39,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:39,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:39,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:39,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +23: [2023-05-13 09:38:39,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:39,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:39,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:39,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:39,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:39,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:39,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:39,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:39,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:39,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:39,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:39,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:39,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:39,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:39,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:39,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +17: [2023-05-13 09:38:39,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +17: [2023-05-13 09:38:39,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +17: [2023-05-13 09:38:39,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:39,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +28: [2023-05-13 09:38:39,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:39,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +28: [2023-05-13 09:38:39,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:39,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:39,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:39,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +14: [2023-05-13 09:38:39,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +27: [2023-05-13 09:38:39,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:39,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +22: [2023-05-13 09:38:39,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +22: [2023-05-13 09:38:39,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +27: [2023-05-13 09:38:39,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:39,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +27: [2023-05-13 09:38:39,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:39,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +22: [2023-05-13 09:38:39,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:39,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:39,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:39,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +27: [2023-05-13 09:38:39,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +27: [2023-05-13 09:38:39,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +27: [2023-05-13 09:38:39,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:39,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:39,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:39,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +17: [2023-05-13 09:38:39,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +21: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:39,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:39,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:39,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +26: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:39,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +26: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:39,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:39,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:39,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:39,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:39,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +23: [2023-05-13 09:38:39,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:39,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:39,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:39,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +21: [2023-05-13 09:38:39,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +21: [2023-05-13 09:38:39,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:39,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:39,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:39,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:39,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:39,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:39,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:39,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:39,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:39,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:39,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:39,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:39,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:39,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:39,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:39,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:39,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:39,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:39,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:39,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:39,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +23: [2023-05-13 09:38:39,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:39,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +10: [2023-05-13 09:38:39,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:39,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:39,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:39,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +23: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:39,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:39,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:39,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:39,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 9: [2023-05-13 09:38:39,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:39,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:39,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:39,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:39,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:39,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:39,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:39,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +31: [2023-05-13 09:38:39,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +12: [2023-05-13 09:38:39,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:39,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:39,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +12: [2023-05-13 09:38:39,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:39,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +13: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +13: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +27: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:39,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:39,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +30: [2023-05-13 09:38:39,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:39,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +30: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +30: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:39,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +30: [2023-05-13 09:38:39,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:39,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +26: [2023-05-13 09:38:39,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +17: [2023-05-13 09:38:39,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +17: [2023-05-13 09:38:39,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:39,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:39,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:39,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:39,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:39,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +17: [2023-05-13 09:38:39,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:39,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:39,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:39,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:39,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:39,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:39,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:39,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:39,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:39,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:39,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +19: [2023-05-13 09:38:39,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:39,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +13: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:39,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +19: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:39,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:39,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:39,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:39,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:39,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:39,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:39,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +10: [2023-05-13 09:38:39,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:39,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:39,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:39,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +12: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:39,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:39,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +22: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:39,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:39,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:39,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:39,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:39,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:39,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:39,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:39,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:39,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:39,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:39,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:39,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:39,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:39,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:39,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +12: [2023-05-13 09:38:39,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:39,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:39,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:39,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:39,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:39,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:39,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:39,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:39,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:39,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +24: [2023-05-13 09:38:39,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:39,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:39,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +24: [2023-05-13 09:38:39,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:39,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:39,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:39,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +11: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:39,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +11: [2023-05-13 09:38:39,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:39,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:39,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:39,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:39,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:39,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:39,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:39,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:39,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:39,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +16: [2023-05-13 09:38:39,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +17: [2023-05-13 09:38:39,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:39,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:39,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:39,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:39,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:39,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:39,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:39,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:39,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:39,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:39,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:39,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:39,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:39,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:39,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:39,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:39,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:39,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:39,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:39,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:39,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:39,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +13: [2023-05-13 09:38:39,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:39,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:39,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:39,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:39,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:39,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:39,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:39,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:39,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:39,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:39,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:39,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:39,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:39,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:39,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:39,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:39,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +13: [2023-05-13 09:38:39,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:39,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:39,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 9: [2023-05-13 09:38:39,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:39,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +13: [2023-05-13 09:38:39,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +24: [2023-05-13 09:38:39,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:39,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:39,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:39,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:39,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:39,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:39,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:39,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:39,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:39,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:39,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +18: [2023-05-13 09:38:39,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:39,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:39,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:39,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:39,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:39,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:39,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:39,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:39,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:39,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:39,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:39,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:39,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +23: [2023-05-13 09:38:39,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:39,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:39,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:39,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:39,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:39,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:39,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:39,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +24: [2023-05-13 09:38:39,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:39,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +24: [2023-05-13 09:38:39,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +24: [2023-05-13 09:38:39,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:39,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:39,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:39,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +10: [2023-05-13 09:38:39,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:39,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +10: [2023-05-13 09:38:39,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:39,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +26: [2023-05-13 09:38:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +26: [2023-05-13 09:38:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +26: [2023-05-13 09:38:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +20: [2023-05-13 09:38:39,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +20: [2023-05-13 09:38:39,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +20: [2023-05-13 09:38:39,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +20: [2023-05-13 09:38:39,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:39,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +18: [2023-05-13 09:38:39,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:39,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:39,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:39,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:39,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +10: [2023-05-13 09:38:39,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:39,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +18: [2023-05-13 09:38:39,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:39,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:39,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:39,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:39,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:39,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:39,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:39,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:39,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:39,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:39,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:39,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:39,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:39,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:39,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:39,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:39,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:39,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:39,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:39,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:39,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:39,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:39,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:39,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:39,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:39,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:39,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:39,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:39,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:39,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:39,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:39,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:39,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:39,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:39,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:39,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:39,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:39,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:39,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:39,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:39,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:39,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:39,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:39,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +25: [2023-05-13 09:38:39,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:39,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +15: [2023-05-13 09:38:39,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:39,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:39,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:39,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:39,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:39,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:39,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:39,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:39,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:39,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:39,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:39,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:39,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:39,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:39,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:39,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:39,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:39,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:39,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:39,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:39,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:39,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:39,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:39,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:39,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +25: [2023-05-13 09:38:39,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:39,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +21: [2023-05-13 09:38:39,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +21: [2023-05-13 09:38:39,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:39,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:39,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:39,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:39,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +11: [2023-05-13 09:38:39,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:39,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:39,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:39,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:39,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:39,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:39,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:39,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:39,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:39,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:39,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:39,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:39,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:39,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:39,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:39,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +21: [2023-05-13 09:38:39,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:39,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +25: [2023-05-13 09:38:39,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:39,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:39,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:39,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:39,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:39,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:39,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:39,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +26: [2023-05-13 09:38:39,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +26: [2023-05-13 09:38:39,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:39,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:39,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +21: [2023-05-13 09:38:39,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:39,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +19: [2023-05-13 09:38:39,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:39,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:39,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:39,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:39,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +19: [2023-05-13 09:38:39,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:39,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:39,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:39,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:39,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:39,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:39,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:39,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:39,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:39,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:39,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:39,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:39,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:39,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:39,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +27: [2023-05-13 09:38:39,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:39,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:39,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:39,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:39,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:39,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:39,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:39,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:39,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:39,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:39,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:39,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 9: [2023-05-13 09:38:39,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:39,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:39,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:39,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:39,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:39,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:39,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:39,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:39,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:39,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:39,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:39,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:39,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:39,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:39,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:39,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:39,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:39,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:39,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:39,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +27: [2023-05-13 09:38:39,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:39,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +27: [2023-05-13 09:38:39,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +27: [2023-05-13 09:38:39,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +27: [2023-05-13 09:38:39,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:39,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:39,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:39,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:39,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:39,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:39,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:39,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:39,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:39,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:39,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:39,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:39,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:39,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:39,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:39,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:39,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:39,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +27: [2023-05-13 09:38:39,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:39,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:39,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:39,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:39,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:39,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:39,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:39,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:39,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:39,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:39,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:39,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:39,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:39,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:39,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:39,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:39,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +13: [2023-05-13 09:38:39,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:39,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:39,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:39,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:39,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:39,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:39,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:39,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:39,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:39,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:39,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:39,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:39,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:39,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:39,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:39,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:39,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:39,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:39,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +27: [2023-05-13 09:38:39,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +27: [2023-05-13 09:38:39,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:39,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:39,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:39,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:39,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:39,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +22: [2023-05-13 09:38:39,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +24: [2023-05-13 09:38:39,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +22: [2023-05-13 09:38:39,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:39,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:39,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:39,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:39,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:39,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:39,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:39,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:39,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:39,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:39,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:39,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:39,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:39,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:39,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:39,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:39,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 9: [2023-05-13 09:38:39,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:39,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:39,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:39,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +14: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:39,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:39,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:39,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:39,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:39,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:39,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +15: [2023-05-13 09:38:39,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +15: [2023-05-13 09:38:39,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:39,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:39,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:39,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:39,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:39,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:39,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:39,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:39,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:39,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:39,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:39,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:39,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:39,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:39,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:39,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:39,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:39,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:39,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:39,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:39,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:39,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:39,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:39,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:39,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +13: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +26: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +26: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:39,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +13: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +26: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +10: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:39,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:39,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:39,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:39,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:39,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:39,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:39,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:39,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:39,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:39,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:39,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:39,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:39,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:39,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:39,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:39,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:39,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:39,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:39,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:39,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +11: [2023-05-13 09:38:39,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:39,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:39,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:39,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:39,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:39,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:39,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:39,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +15: [2023-05-13 09:38:39,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:39,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:39,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:39,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:39,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:39,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:39,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:39,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:39,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:39,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:39,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:39,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:39,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:39,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:39,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:39,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:39,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:39,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:39,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:39,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:39,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:39,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +31: [2023-05-13 09:38:39,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:39,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +20: [2023-05-13 09:38:39,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +20: [2023-05-13 09:38:39,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:39,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:39,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:39,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:39,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +10: [2023-05-13 09:38:39,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +15: [2023-05-13 09:38:39,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:39,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:39,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +10: [2023-05-13 09:38:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:39,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:39,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:39,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:39,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:39,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:39,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +14: [2023-05-13 09:38:39,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:39,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:39,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:39,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +14: [2023-05-13 09:38:39,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +14: [2023-05-13 09:38:39,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:39,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:39,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +13: [2023-05-13 09:38:39,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +20: [2023-05-13 09:38:39,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:39,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +13: [2023-05-13 09:38:39,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +20: [2023-05-13 09:38:39,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +24: [2023-05-13 09:38:39,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +26: [2023-05-13 09:38:39,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +24: [2023-05-13 09:38:39,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:39,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:39,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +26: [2023-05-13 09:38:39,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +16: [2023-05-13 09:38:39,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +16: [2023-05-13 09:38:39,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +21: [2023-05-13 09:38:39,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:39,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:39,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +16: [2023-05-13 09:38:39,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. +26: [2023-05-13 09:38:39,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:39,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:39,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:39,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:39,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +21: [2023-05-13 09:38:39,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:39,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:39,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +13: [2023-05-13 09:38:39,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:39,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:39,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:39,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:39,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:39,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:39,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:39,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:39,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:39,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:39,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +20: [2023-05-13 09:38:39,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +20: [2023-05-13 09:38:39,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:39,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:39,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:39,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:39,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:39,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:39,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +20: [2023-05-13 09:38:39,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:39,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:39,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 9: [2023-05-13 09:38:39,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:39,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:39,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +21: [2023-05-13 09:38:39,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +21: [2023-05-13 09:38:39,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 9: [2023-05-13 09:38:39,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:39,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:39,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +13: [2023-05-13 09:38:39,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:39,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +13: [2023-05-13 09:38:39,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +13: [2023-05-13 09:38:39,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:39,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +16: [2023-05-13 09:38:39,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:39,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:39,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:39,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:39,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:39,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:39,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:39,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:39,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +27: [2023-05-13 09:38:39,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:39,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:39,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:39,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:39,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:39,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:39,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:39,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:39,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:39,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:39,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:39,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:39,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:39,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_24-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:39,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:39,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:39,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:39,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:39,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:39,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:39,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:39,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:39,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:39,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:39,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 9: [2023-05-13 09:38:39,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:39,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:39,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:39,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:39,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:39,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:39,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:39,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:39,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:39,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:39,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:39,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:39,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +25: [2023-05-13 09:38:39,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:39,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:39,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:39,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:39,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +19: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +19: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +19: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:39,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:39,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:39,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:39,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:39,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:39,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:39,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:39,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +17: [2023-05-13 09:38:39,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:39,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:39,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:39,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:39,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:39,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +26: [2023-05-13 09:38:39,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:39,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:39,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:39,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:39,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:39,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +28: [2023-05-13 09:38:39,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +28: [2023-05-13 09:38:39,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:39,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:39,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:39,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +23: [2023-05-13 09:38:39,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:39,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:39,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:39,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:39,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:39,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:39,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:39,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:39,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:39,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:39,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:39,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:39,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +29: [2023-05-13 09:38:39,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:39,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:39,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:39,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:39,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:39,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:39,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:39,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:39,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:39,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:39,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:39,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:39,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:39,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:39,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:39,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +17: [2023-05-13 09:38:39,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:39,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:39,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:39,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:39,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:39,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:39,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:39,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:39,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:39,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:39,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:39,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:39,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:39,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:39,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:39,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:39,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:39,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:39,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:39,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:39,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:39,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:39,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:39,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:39,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:39,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:39,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:39,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:39,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:39,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:39,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:39,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:39,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +27: [2023-05-13 09:38:39,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +27: [2023-05-13 09:38:39,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +15: [2023-05-13 09:38:39,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:39,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +20: [2023-05-13 09:38:39,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:39,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +12: [2023-05-13 09:38:39,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +15: [2023-05-13 09:38:39,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:39,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:39,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:39,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:39,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:39,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:39,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:39,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:39,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:39,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:39,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:39,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:39,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:39,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:39,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:39,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:39,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:39,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:39,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:39,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +12: [2023-05-13 09:38:39,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:39,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:39,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:39,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:39,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:39,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:39,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +29: [2023-05-13 09:38:39,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +29: [2023-05-13 09:38:39,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +22: [2023-05-13 09:38:39,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:39,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:39,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:39,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:39,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:39,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:39,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:39,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:39,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:39,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:39,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +17: [2023-05-13 09:38:39,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:39,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:39,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:39,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:39,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:39,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:39,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:39,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:39,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +10: [2023-05-13 09:38:39,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:39,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:39,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +19: [2023-05-13 09:38:39,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:39,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:39,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:39,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:39,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:39,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:39,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:39,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:39,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +15: [2023-05-13 09:38:39,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:39,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +10: [2023-05-13 09:38:39,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:39,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:39,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:39,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:39,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:39,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:39,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:39,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:39,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:39,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:39,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:39,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:39,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:39,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:39,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:39,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:40,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:39,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +15: [2023-05-13 09:38:40,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:40,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +27: [2023-05-13 09:38:40,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:40,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:40,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:40,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:40,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:40,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:40,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:40,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:40,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:40,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:40,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:40,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:40,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:40,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:40,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:40,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:40,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:40,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:40,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +14: [2023-05-13 09:38:40,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:40,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:40,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:40,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:40,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:40,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:40,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:40,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:40,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:40,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:40,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:40,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:40,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:40,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:40,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:40,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +15: [2023-05-13 09:38:40,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:40,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +24: [2023-05-13 09:38:40,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +19: [2023-05-13 09:38:40,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:40,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +24: [2023-05-13 09:38:40,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:40,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:40,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:40,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:40,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:40,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:40,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:40,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:40,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:40,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:40,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:40,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:40,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:40,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:40,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:40,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:40,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:40,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:40,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:40,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:40,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:40,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:40,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:40,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:40,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:40,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +29: [2023-05-13 09:38:40,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:40,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:40,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:40,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:40,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:40,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:40,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:40,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:40,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +23: [2023-05-13 09:38:40,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:40,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:40,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:40,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:40,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:40,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:40,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:40,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +25: [2023-05-13 09:38:40,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +18: [2023-05-13 09:38:40,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:40,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:40,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:40,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +25: [2023-05-13 09:38:40,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:40,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +15: [2023-05-13 09:38:40,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:40,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:40,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:40,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +31: [2023-05-13 09:38:40,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +31: [2023-05-13 09:38:40,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:40,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:40,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +29: [2023-05-13 09:38:40,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +14: [2023-05-13 09:38:40,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:40,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:40,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:40,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:40,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:40,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:40,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +14: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +14: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:40,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +15: [2023-05-13 09:38:40,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +15: [2023-05-13 09:38:40,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:40,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:40,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:40,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:40,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:40,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:40,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:40,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:40,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:40,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:40,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 4: [2023-05-13 09:38:40,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:40,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 4: [2023-05-13 09:38:40,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +30: [2023-05-13 09:38:40,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +30: [2023-05-13 09:38:40,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +30: [2023-05-13 09:38:40,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:40,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:40,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:40,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:40,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:40,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:40,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:40,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:40,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:40,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:40,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:40,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:40,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +10: [2023-05-13 09:38:40,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +10: [2023-05-13 09:38:40,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:40,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:40,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:40,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +11: [2023-05-13 09:38:40,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:40,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:40,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:40,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:40,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +20: [2023-05-13 09:38:40,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:40,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:40,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:40,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +31: [2023-05-13 09:38:40,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:40,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:40,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:40,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:40,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:40,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:40,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +11: [2023-05-13 09:38:40,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:40,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +26: [2023-05-13 09:38:40,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +26: [2023-05-13 09:38:40,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +11: [2023-05-13 09:38:40,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:40,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +25: [2023-05-13 09:38:40,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:40,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:40,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +25: [2023-05-13 09:38:40,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +11: [2023-05-13 09:38:40,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:40,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:40,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:40,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:40,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:40,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:40,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:40,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:40,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +20: [2023-05-13 09:38:40,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:40,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:40,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:40,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +25: [2023-05-13 09:38:40,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:40,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +21: [2023-05-13 09:38:40,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +21: [2023-05-13 09:38:40,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +21: [2023-05-13 09:38:40,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +28: [2023-05-13 09:38:40,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:40,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:40,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:40,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +13: [2023-05-13 09:38:40,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +26: [2023-05-13 09:38:40,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +16: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +13: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +13: [2023-05-13 09:38:40,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:40,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:40,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:40,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +28: [2023-05-13 09:38:40,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:40,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:40,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:40,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:40,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:40,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:40,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:40,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:40,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:40,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:40,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +20: [2023-05-13 09:38:40,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +16: [2023-05-13 09:38:40,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:40,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:40,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:40,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:40,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:40,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:40,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:40,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:40,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +13: [2023-05-13 09:38:40,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:40,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:40,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:40,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:40,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +20: [2023-05-13 09:38:40,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:40,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:40,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:40,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:40,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:40,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:40,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:40,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:40,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:40,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:40,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +18: [2023-05-13 09:38:40,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +18: [2023-05-13 09:38:40,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +18: [2023-05-13 09:38:40,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +21: [2023-05-13 09:38:40,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:40,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:40,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:40,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:40,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:40,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:40,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:40,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +16: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:40,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:40,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:40,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:40,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:40,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +13: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +16: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +16: [2023-05-13 09:38:40,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +13: [2023-05-13 09:38:40,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +13: [2023-05-13 09:38:40,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:40,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:40,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:40,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:40,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:40,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:40,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:40,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:40,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:40,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:40,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:40,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:40,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:40,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:40,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:40,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:40,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:40,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:40,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 4: [2023-05-13 09:38:40,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +23: [2023-05-13 09:38:40,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +23: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:40,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:40,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:40,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:40,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:40,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:40,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +16: [2023-05-13 09:38:40,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +16: [2023-05-13 09:38:40,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:40,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:40,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:40,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:40,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:40,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +23: [2023-05-13 09:38:40,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:40,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:40,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +29: [2023-05-13 09:38:40,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:40,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:40,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +28: [2023-05-13 09:38:40,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +29: [2023-05-13 09:38:40,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:40,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +28: [2023-05-13 09:38:40,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:40,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:40,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:40,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:40,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:40,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:40,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:40,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:40,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +12: [2023-05-13 09:38:40,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +12: [2023-05-13 09:38:40,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +12: [2023-05-13 09:38:40,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +29: [2023-05-13 09:38:40,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:40,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:40,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:40,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:40,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:40,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:40,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:40,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +29: [2023-05-13 09:38:40,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:40,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:40,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:40,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:40,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:40,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +22: [2023-05-13 09:38:40,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +28: [2023-05-13 09:38:40,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +28: [2023-05-13 09:38:40,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:40,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:40,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:40,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:40,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +28: [2023-05-13 09:38:40,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:40,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +28: [2023-05-13 09:38:40,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +28: [2023-05-13 09:38:40,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +12: [2023-05-13 09:38:40,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:40,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:40,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:40,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:40,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:40,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:40,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +17: [2023-05-13 09:38:40,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:40,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:40,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:40,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:40,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:40,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:40,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:40,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:40,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:40,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:40,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:40,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:40,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:40,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:40,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:40,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:40,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:40,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:40,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:40,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:40,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:40,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:40,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:40,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:40,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +12: [2023-05-13 09:38:40,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +12: [2023-05-13 09:38:40,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:40,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:40,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:40,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:40,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:40,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +14: [2023-05-13 09:38:40,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:40,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:40,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +22: [2023-05-13 09:38:40,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +22: [2023-05-13 09:38:40,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +29: [2023-05-13 09:38:40,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:40,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +14: [2023-05-13 09:38:40,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:40,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:40,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:40,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:40,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:40,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:40,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:40,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:40,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:40,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:40,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:40,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:40,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:40,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +14: [2023-05-13 09:38:40,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:40,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +18: [2023-05-13 09:38:40,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +18: [2023-05-13 09:38:40,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:40,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +18: [2023-05-13 09:38:40,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:40,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:40,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:40,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:40,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:40,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:40,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:40,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:40,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +17: [2023-05-13 09:38:40,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:40,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +31: [2023-05-13 09:38:40,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:40,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +23: [2023-05-13 09:38:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +23: [2023-05-13 09:38:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +23: [2023-05-13 09:38:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +18: [2023-05-13 09:38:40,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:40,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +31: [2023-05-13 09:38:40,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +31: [2023-05-13 09:38:40,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:40,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +19: [2023-05-13 09:38:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +19: [2023-05-13 09:38:40,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +19: [2023-05-13 09:38:40,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +31: [2023-05-13 09:38:40,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:40,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:40,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:40,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:40,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:40,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:40,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:40,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:40,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +11: [2023-05-13 09:38:40,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +22: [2023-05-13 09:38:40,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:40,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:40,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:40,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:40,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:40,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:40,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:40,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:40,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +21: [2023-05-13 09:38:40,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +21: [2023-05-13 09:38:40,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +13: [2023-05-13 09:38:40,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:40,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:40,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:40,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:40,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:40,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:40,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:40,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:40,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:40,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:40,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:40,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:40,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:40,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:40,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:40,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:40,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:40,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:40,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:40,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:40,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:40,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:40,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:40,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:40,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:40,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:40,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:40,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:40,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:40,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:40,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:40,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +31: [2023-05-13 09:38:40,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +19: [2023-05-13 09:38:40,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:40,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:40,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +31: [2023-05-13 09:38:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:40,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +18: [2023-05-13 09:38:40,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:40,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:40,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:40,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +14: [2023-05-13 09:38:40,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:40,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:40,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:40,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:40,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:40,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:40,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:40,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:40,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:40,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:40,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:40,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:40,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:40,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:40,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +19: [2023-05-13 09:38:40,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:40,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:40,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +30: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +19: [2023-05-13 09:38:40,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:40,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:40,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:40,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:40,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:40,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:40,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:40,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:40,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:40,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:40,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:40,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:40,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +20: [2023-05-13 09:38:40,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +20: [2023-05-13 09:38:40,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +20: [2023-05-13 09:38:40,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +15: [2023-05-13 09:38:40,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:40,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:40,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:40,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:40,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:40,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:40,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:40,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:40,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +14: [2023-05-13 09:38:40,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +14: [2023-05-13 09:38:40,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:40,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:40,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:40,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:40,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:40,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:40,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:40,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:40,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:40,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:40,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:40,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:40,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:40,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:40,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:40,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:40,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:40,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:40,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:40,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:40,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +15: [2023-05-13 09:38:40,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +16: [2023-05-13 09:38:40,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +16: [2023-05-13 09:38:40,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +21: [2023-05-13 09:38:40,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +21: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +15: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +16: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +13: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +13: [2023-05-13 09:38:40,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:40,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:40,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +21: [2023-05-13 09:38:40,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +21: [2023-05-13 09:38:40,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +24: [2023-05-13 09:38:40,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:40,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:40,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:40,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:40,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +13: [2023-05-13 09:38:40,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:40,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:40,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:40,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:40,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +19: [2023-05-13 09:38:40,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:40,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:40,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:40,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:40,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +26: [2023-05-13 09:38:40,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +26: [2023-05-13 09:38:40,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +19: [2023-05-13 09:38:40,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:40,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:40,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:40,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:40,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:40,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:40,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:40,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:40,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +25: [2023-05-13 09:38:40,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +25: [2023-05-13 09:38:40,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 4: [2023-05-13 09:38:40,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +25: [2023-05-13 09:38:40,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +25: [2023-05-13 09:38:40,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 4: [2023-05-13 09:38:40,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:40,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +30: [2023-05-13 09:38:40,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:40,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:40,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +18: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:40,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:40,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +18: [2023-05-13 09:38:40,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:40,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +18: [2023-05-13 09:38:40,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:40,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:40,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:40,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:40,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:40,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:40,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +20: [2023-05-13 09:38:40,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +25: [2023-05-13 09:38:40,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +30: [2023-05-13 09:38:40,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:40,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:40,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:40,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:40,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:40,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:40,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... +20: [2023-05-13 09:38:40,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:40,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:40,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:40,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:40,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:40,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +25: [2023-05-13 09:38:40,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +25: [2023-05-13 09:38:40,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +25: [2023-05-13 09:38:40,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +25: [2023-05-13 09:38:40,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +25: [2023-05-13 09:38:40,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:40,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:40,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:40,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:40,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:40,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:40,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +29: [2023-05-13 09:38:40,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:40,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +17: [2023-05-13 09:38:40,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +17: [2023-05-13 09:38:40,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +17: [2023-05-13 09:38:40,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +17: [2023-05-13 09:38:40,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +23: [2023-05-13 09:38:40,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:40,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +10: [2023-05-13 09:38:40,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +10: [2023-05-13 09:38:40,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +10: [2023-05-13 09:38:40,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +23: [2023-05-13 09:38:40,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:40,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:40,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:40,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:40,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +29: [2023-05-13 09:38:40,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:40,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:40,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:40,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:40,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:40,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:40,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +10: [2023-05-13 09:38:40,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:40,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +23: [2023-05-13 09:38:40,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:40,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:40,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:40,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:40,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:40,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:40,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:40,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:40,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:40,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:40,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:40,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:40,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:40,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:40,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:40,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:40,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +12: [2023-05-13 09:38:40,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +12: [2023-05-13 09:38:40,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +12: [2023-05-13 09:38:40,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:40,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:40,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:40,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +17: [2023-05-13 09:38:40,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:40,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:40,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +10: [2023-05-13 09:38:40,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +10: [2023-05-13 09:38:40,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:40,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:40,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:40,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:40,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:40,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:40,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:40,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:40,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:40,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:40,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:40,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:40,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:40,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:40,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:40,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:40,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:40,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:40,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:40,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:40,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:40,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:40,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:40,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:40,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:40,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:40,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:40,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:40,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:40,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:40,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:40,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:40,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:40,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:40,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:40,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:40,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:40,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:40,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:40,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:40,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:40,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:40,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:40,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:40,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:40,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:40,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:40,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:40,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +13: [2023-05-13 09:38:40,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:40,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +12: [2023-05-13 09:38:40,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +25: [2023-05-13 09:38:40,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +19: [2023-05-13 09:38:40,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:40,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:40,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:40,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:40,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:40,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:40,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:40,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +17: [2023-05-13 09:38:40,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:40,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:40,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:40,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:40,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:40,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:40,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:40,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:40,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:40,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:40,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:40,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:40,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:40,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:40,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:40,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:40,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:40,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:40,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:40,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +12: [2023-05-13 09:38:40,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:40,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:40,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:40,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:40,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:40,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:40,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:40,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:40,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:40,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:40,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:40,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:40,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:40,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:40,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +19: [2023-05-13 09:38:40,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:40,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:40,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:40,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:40,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:40,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:40,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:40,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:40,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:40,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:40,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:40,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:40,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:40,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:40,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +23: [2023-05-13 09:38:40,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +23: [2023-05-13 09:38:40,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +23: [2023-05-13 09:38:40,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +20: [2023-05-13 09:38:40,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:40,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:40,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:40,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:40,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:40,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:40,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:40,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +24: [2023-05-13 09:38:40,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +11: [2023-05-13 09:38:40,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:40,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:40,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +15: [2023-05-13 09:38:40,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:40,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:40,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +15: [2023-05-13 09:38:40,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +19: [2023-05-13 09:38:40,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +19: [2023-05-13 09:38:40,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +19: [2023-05-13 09:38:40,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +19: [2023-05-13 09:38:40,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:40,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +14: [2023-05-13 09:38:40,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:40,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:40,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:40,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:40,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:40,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +23: [2023-05-13 09:38:40,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:40,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +23: [2023-05-13 09:38:40,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:40,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:40,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:40,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:40,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:40,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:40,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:40,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:40,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:40,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:40,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:40,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:40,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:40,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:40,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:40,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +13: [2023-05-13 09:38:40,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:40,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:40,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:40,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:40,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +14: [2023-05-13 09:38:40,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:40,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:40,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:40,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:40,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:40,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +24: [2023-05-13 09:38:40,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +24: [2023-05-13 09:38:40,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:40,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:40,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:40,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:40,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:40,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:40,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:40,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:40,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +23: [2023-05-13 09:38:40,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +27: [2023-05-13 09:38:40,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:40,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:40,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +11: [2023-05-13 09:38:40,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +13: [2023-05-13 09:38:40,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:40,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:40,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:40,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:40,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:40,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:40,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:40,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:40,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:40,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +11: [2023-05-13 09:38:40,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:40,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:40,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:40,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:40,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:40,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:40,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:40,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:40,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:40,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:40,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:40,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:40,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +27: [2023-05-13 09:38:40,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:40,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +27: [2023-05-13 09:38:40,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:40,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:40,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:40,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +27: [2023-05-13 09:38:40,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +13: [2023-05-13 09:38:40,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +13: [2023-05-13 09:38:40,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +13: [2023-05-13 09:38:40,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +13: [2023-05-13 09:38:40,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:41,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:41,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:41,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +12: [2023-05-13 09:38:41,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:41,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:41,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:41,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:41,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 2: [2023-05-13 09:38:41,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 2: [2023-05-13 09:38:41,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 2: [2023-05-13 09:38:41,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:41,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:41,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:41,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:41,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:41,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:41,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:41,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:41,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:41,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:41,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:41,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:41,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:41,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:41,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:41,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:41,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:41,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:41,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:41,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +24: [2023-05-13 09:38:41,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:41,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:41,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +24: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:41,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:41,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:41,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +25: [2023-05-13 09:38:41,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 2: [2023-05-13 09:38:41,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +25: [2023-05-13 09:38:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:41,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:41,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +27: [2023-05-13 09:38:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:41,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:41,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:41,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:41,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:41,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:41,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:41,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:41,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:41,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:41,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:41,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:41,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +21: [2023-05-13 09:38:41,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:41,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:41,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:41,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:41,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:41,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:41,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:41,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:41,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:41,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:41,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:41,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:41,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +12: [2023-05-13 09:38:41,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +12: [2023-05-13 09:38:41,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:41,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:41,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:41,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:41,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:41,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:41,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:41,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:41,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:41,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:41,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:41,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:41,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:41,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:41,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:41,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +27: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +12: [2023-05-13 09:38:41,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:41,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:41,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:41,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:41,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:41,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:41,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:41,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:41,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:41,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:41,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:41,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:41,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:41,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:41,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:41,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:41,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:41,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:41,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:41,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:41,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:41,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:41,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:41,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:41,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:41,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:41,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:41,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +29: [2023-05-13 09:38:41,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:41,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:41,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:41,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:41,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:41,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:41,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:41,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:41,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:41,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:41,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:41,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +15: [2023-05-13 09:38:41,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:41,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:41,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:41,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:41,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:41,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +15: [2023-05-13 09:38:41,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:41,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:41,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +18: [2023-05-13 09:38:41,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:41,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:41,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:41,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:41,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:41,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:41,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:41,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:41,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:41,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:41,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:41,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +12: [2023-05-13 09:38:41,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +18: [2023-05-13 09:38:41,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:41,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:41,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +21: [2023-05-13 09:38:41,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:41,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:41,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:41,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:41,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:41,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:41,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:41,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:41,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:41,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +31: [2023-05-13 09:38:41,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:41,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +26: [2023-05-13 09:38:41,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +13: [2023-05-13 09:38:41,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:41,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:41,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +25: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +16: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +31: [2023-05-13 09:38:41,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:41,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:41,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:41,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:41,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:41,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:41,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:41,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:41,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:41,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:41,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:41,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:41,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:41,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:41,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:41,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:41,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:41,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:41,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:41,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:41,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:41,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:41,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:41,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:41,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:41,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:41,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:41,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:41,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:41,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:41,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:41,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +28: [2023-05-13 09:38:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +28: [2023-05-13 09:38:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +28: [2023-05-13 09:38:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +28: [2023-05-13 09:38:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:41,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:41,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:41,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +26: [2023-05-13 09:38:41,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +15: [2023-05-13 09:38:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +15: [2023-05-13 09:38:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +15: [2023-05-13 09:38:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +13: [2023-05-13 09:38:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +13: [2023-05-13 09:38:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +13: [2023-05-13 09:38:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +31: [2023-05-13 09:38:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +31: [2023-05-13 09:38:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +13: [2023-05-13 09:38:41,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +31: [2023-05-13 09:38:41,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +31: [2023-05-13 09:38:41,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +26: [2023-05-13 09:38:41,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:41,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:41,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:41,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:41,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:41,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:41,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:41,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:41,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +27: [2023-05-13 09:38:41,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +27: [2023-05-13 09:38:41,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +27: [2023-05-13 09:38:41,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +27: [2023-05-13 09:38:41,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:41,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:41,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:41,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:41,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:41,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:41,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:41,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:41,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:41,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:41,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:41,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:41,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:41,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:41,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:41,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:41,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:41,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:41,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:41,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:41,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:41,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:41,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:41,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:41,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:41,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:41,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:41,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:41,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:41,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:41,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:41,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:41,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:41,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:41,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:41,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:41,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:41,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:41,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:41,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:41,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:41,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +28: [2023-05-13 09:38:41,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:41,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:41,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:41,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:41,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:41,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +28: [2023-05-13 09:38:41,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:41,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:41,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:41,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:41,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:41,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:41,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +28: [2023-05-13 09:38:41,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +28: [2023-05-13 09:38:41,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:41,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:41,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:41,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:41,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +15: [2023-05-13 09:38:41,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:41,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +15: [2023-05-13 09:38:41,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +15: [2023-05-13 09:38:41,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:41,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:41,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +15: [2023-05-13 09:38:41,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +11: [2023-05-13 09:38:41,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:41,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:41,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:41,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:41,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:41,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +13: [2023-05-13 09:38:41,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:41,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:41,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:41,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +31: [2023-05-13 09:38:41,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +13: [2023-05-13 09:38:41,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:41,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:41,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:41,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:41,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:41,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:41,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:41,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:41,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +11: [2023-05-13 09:38:41,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:41,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:41,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:41,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +11: [2023-05-13 09:38:41,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:41,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:41,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:41,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:41,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:41,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +15: [2023-05-13 09:38:41,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:41,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:41,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +31: [2023-05-13 09:38:41,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:41,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +15: [2023-05-13 09:38:41,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +27: [2023-05-13 09:38:41,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:41,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +27: [2023-05-13 09:38:41,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:41,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +27: [2023-05-13 09:38:41,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:41,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +23: [2023-05-13 09:38:41,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +23: [2023-05-13 09:38:41,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +23: [2023-05-13 09:38:41,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:41,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:41,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:41,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:41,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:41,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:41,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:41,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:41,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:41,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:41,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:41,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:41,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:41,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +21: [2023-05-13 09:38:41,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +21: [2023-05-13 09:38:41,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +21: [2023-05-13 09:38:41,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +21: [2023-05-13 09:38:41,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +24: [2023-05-13 09:38:41,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +24: [2023-05-13 09:38:41,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +24: [2023-05-13 09:38:41,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +17: [2023-05-13 09:38:41,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +17: [2023-05-13 09:38:41,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +24: [2023-05-13 09:38:41,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +17: [2023-05-13 09:38:41,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +17: [2023-05-13 09:38:41,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +16: [2023-05-13 09:38:41,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +16: [2023-05-13 09:38:41,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +16: [2023-05-13 09:38:41,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +16: [2023-05-13 09:38:41,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:41,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:41,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:41,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:41,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:41,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:41,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:41,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:41,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:41,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:41,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:41,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:41,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:41,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:41,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:41,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +24: [2023-05-13 09:38:41,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:41,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:41,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:41,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +24: [2023-05-13 09:38:41,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:41,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:41,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:41,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:41,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:41,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:41,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:41,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:41,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:41,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +24: [2023-05-13 09:38:41,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +17: [2023-05-13 09:38:41,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:41,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:41,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:41,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +17: [2023-05-13 09:38:41,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:41,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +21: [2023-05-13 09:38:41,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:41,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:41,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +17: [2023-05-13 09:38:41,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +17: [2023-05-13 09:38:41,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:41,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:41,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:41,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:41,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:41,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:41,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:41,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:41,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:41,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +29: [2023-05-13 09:38:41,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +29: [2023-05-13 09:38:41,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +29: [2023-05-13 09:38:41,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:41,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:41,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:41,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +31: [2023-05-13 09:38:41,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:41,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +31: [2023-05-13 09:38:41,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:41,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +31: [2023-05-13 09:38:41,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +31: [2023-05-13 09:38:41,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:41,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:41,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:41,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:41,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:41,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:41,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:41,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:41,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +27: [2023-05-13 09:38:41,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:41,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +31: [2023-05-13 09:38:41,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:41,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:41,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +31: [2023-05-13 09:38:41,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +31: [2023-05-13 09:38:41,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:41,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +14: [2023-05-13 09:38:41,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:41,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:41,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:41,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:41,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:41,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:41,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:41,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:41,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:41,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:41,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:41,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:41,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:41,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:41,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:41,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +27: [2023-05-13 09:38:41,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:41,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:41,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:41,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:41,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:41,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:41,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:41,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:41,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:41,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:41,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:41,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:41,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:41,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:41,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +15: [2023-05-13 09:38:41,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:41,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:41,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:41,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:41,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:41,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:41,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:41,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:41,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:41,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:41,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:41,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:41,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:41,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:41,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:41,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:41,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:41,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:41,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:41,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:41,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:41,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:41,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:41,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:41,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:41,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:41,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:41,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:41,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:41,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:41,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:41,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:41,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:41,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:41,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:41,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:41,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:41,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +15: [2023-05-13 09:38:41,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:41,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:41,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:41,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:41,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:41,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:41,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:41,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:41,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:41,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:41,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:41,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:41,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +26: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +11: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:41,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:41,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:41,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:41,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:41,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:41,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:41,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:41,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:41,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:41,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:41,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:41,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +24: [2023-05-13 09:38:41,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:41,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:41,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:41,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:41,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:41,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:41,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:41,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:41,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:41,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +10: [2023-05-13 09:38:41,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:41,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:41,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:41,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:41,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... +11: [2023-05-13 09:38:41,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:41,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:41,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:41,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:41,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:41,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:41,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:41,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:41,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:41,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:41,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:41,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:41,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:41,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:41,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:41,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:41,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:41,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:41,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:41,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:41,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:41,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:41,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:41,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:41,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:41,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:41,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:41,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:41,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +12: [2023-05-13 09:38:41,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +12: [2023-05-13 09:38:41,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +12: [2023-05-13 09:38:41,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +28: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +11: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +11: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +11: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +11: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +28: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +25: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +25: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +15: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:41,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +15: [2023-05-13 09:38:41,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:41,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:41,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:41,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:41,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:41,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:41,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:41,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:41,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:41,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:41,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:41,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:41,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:41,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:41,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:41,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:41,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:41,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:41,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:41,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:41,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:41,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:41,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:41,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:41,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:41,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:41,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:41,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:41,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:41,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:41,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:41,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:41,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:41,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:41,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:41,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:41,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:41,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:41,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:41,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:41,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:41,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:41,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:41,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:41,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:41,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:41,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:41,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:41,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:41,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +11: [2023-05-13 09:38:41,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:41,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +12: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +28: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:41,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:41,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:41,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:41,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +26: [2023-05-13 09:38:41,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +14: [2023-05-13 09:38:41,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +12: [2023-05-13 09:38:41,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +14: [2023-05-13 09:38:41,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +11: [2023-05-13 09:38:41,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:41,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +12: [2023-05-13 09:38:41,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +28: [2023-05-13 09:38:41,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:41,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:41,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:41,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +28: [2023-05-13 09:38:41,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +25: [2023-05-13 09:38:41,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:41,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:41,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:41,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:41,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +25: [2023-05-13 09:38:41,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +26: [2023-05-13 09:38:41,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:41,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:41,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +28: [2023-05-13 09:38:41,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:41,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:41,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +25: [2023-05-13 09:38:41,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:41,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:41,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:41,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:41,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:41,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:41,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:41,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +13: [2023-05-13 09:38:41,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:41,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:41,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +15: [2023-05-13 09:38:41,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:41,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:41,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +10: [2023-05-13 09:38:41,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +10: [2023-05-13 09:38:41,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +10: [2023-05-13 09:38:41,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:41,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:41,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:41,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:41,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:41,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +27: [2023-05-13 09:38:41,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:41,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:41,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +27: [2023-05-13 09:38:41,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +27: [2023-05-13 09:38:41,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:41,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:41,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:41,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:41,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:41,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:41,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:41,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:41,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:41,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:41,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:41,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:41,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:41,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:41,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:41,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:41,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:41,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:41,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:41,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:41,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:41,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:41,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:41,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:41,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:41,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:41,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:41,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:41,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:41,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +22: [2023-05-13 09:38:41,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +22: [2023-05-13 09:38:41,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +22: [2023-05-13 09:38:41,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:41,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:41,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:41,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:41,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:41,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:41,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:41,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:41,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:41,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:41,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:41,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:41,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +22: [2023-05-13 09:38:41,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:41,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:41,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:41,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +13: [2023-05-13 09:38:41,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:41,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:41,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +10: [2023-05-13 09:38:41,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +24: [2023-05-13 09:38:41,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:41,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:41,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:41,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +27: [2023-05-13 09:38:41,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +27: [2023-05-13 09:38:41,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +10: [2023-05-13 09:38:41,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +10: [2023-05-13 09:38:41,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +24: [2023-05-13 09:38:41,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:41,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:41,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +24: [2023-05-13 09:38:41,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +24: [2023-05-13 09:38:41,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +24: [2023-05-13 09:38:41,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +27: [2023-05-13 09:38:41,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:42,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +29: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +29: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +29: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +30: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +30: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +30: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +30: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:42,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:42,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +22: [2023-05-13 09:38:42,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:42,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +22: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +30: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:42,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:42,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:42,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:42,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:42,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:42,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:42,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:42,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:42,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:42,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:42,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:42,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:42,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:42,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:42,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +30: [2023-05-13 09:38:42,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:42,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:42,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:42,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:42,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:42,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:42,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:42,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:42,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:42,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +30: [2023-05-13 09:38:42,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:42,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:42,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:42,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:42,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:42,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:42,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +17: [2023-05-13 09:38:42,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +29: [2023-05-13 09:38:42,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:42,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:42,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:42,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:42,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:42,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:42,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:42,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:42,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:42,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:42,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:42,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:42,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:42,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:42,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:42,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:42,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +29: [2023-05-13 09:38:42,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:42,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +16: [2023-05-13 09:38:42,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:42,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:42,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:42,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:42,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:42,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:42,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:42,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +21: [2023-05-13 09:38:42,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:42,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:42,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:42,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:42,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:42,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +17: [2023-05-13 09:38:42,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:42,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:42,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:42,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:42,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:42,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:42,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +16: [2023-05-13 09:38:42,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:42,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:42,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:42,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +14: [2023-05-13 09:38:42,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:42,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:42,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +14: [2023-05-13 09:38:42,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:42,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:42,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:42,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:42,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:42,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:42,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:42,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:42,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:42,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:42,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:42,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:42,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:42,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:42,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:42,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:42,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:42,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:42,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:42,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:42,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:42,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:42,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:42,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:42,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:42,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:42,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:42,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:42,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:42,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:42,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:42,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:42,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:42,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +21: [2023-05-13 09:38:42,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. +31: [2023-05-13 09:38:42,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:42,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +21: [2023-05-13 09:38:42,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +31: [2023-05-13 09:38:42,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:42,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:42,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:42,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +31: [2023-05-13 09:38:42,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +31: [2023-05-13 09:38:42,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:42,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:42,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:42,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +14: [2023-05-13 09:38:42,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:42,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:42,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:42,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:42,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:42,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:42,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:42,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:42,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +12: [2023-05-13 09:38:42,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +12: [2023-05-13 09:38:42,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +12: [2023-05-13 09:38:42,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:42,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +19: [2023-05-13 09:38:42,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +19: [2023-05-13 09:38:42,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:42,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:42,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:42,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:42,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:42,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:42,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:42,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:42,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:42,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:42,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +27: [2023-05-13 09:38:42,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +27: [2023-05-13 09:38:42,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +27: [2023-05-13 09:38:42,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +19: [2023-05-13 09:38:42,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:42,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:42,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:42,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +30: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +30: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +30: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:42,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:42,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:42,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:42,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:42,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:42,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:42,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:42,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:42,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:42,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:42,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:42,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt... +30: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:42,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:42,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:42,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:42,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:42,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:42,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +19: [2023-05-13 09:38:42,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +12: [2023-05-13 09:38:42,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:42,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +12: [2023-05-13 09:38:42,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:42,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +12: [2023-05-13 09:38:42,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:42,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +24: [2023-05-13 09:38:42,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +12: [2023-05-13 09:38:42,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:42,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:42,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:42,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:42,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:42,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:42,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:42,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:42,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:42,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:42,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:42,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:42,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:42,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:42,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:42,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:42,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:42,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:42,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:42,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:42,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:42,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:42,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:42,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:42,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:42,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:42,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:42,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:42,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:42,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:42,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:42,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:42,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +27: [2023-05-13 09:38:42,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +27: [2023-05-13 09:38:42,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +27: [2023-05-13 09:38:42,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:42,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:42,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +30: [2023-05-13 09:38:42,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:42,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:42,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:42,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:42,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:42,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:42,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +27: [2023-05-13 09:38:42,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +27: [2023-05-13 09:38:42,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +23: [2023-05-13 09:38:42,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:42,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:42,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:42,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +30: [2023-05-13 09:38:42,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:42,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +23: [2023-05-13 09:38:42,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +23: [2023-05-13 09:38:42,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:42,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:42,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +15: [2023-05-13 09:38:42,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +15: [2023-05-13 09:38:42,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +15: [2023-05-13 09:38:42,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +15: [2023-05-13 09:38:42,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:42,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +30: [2023-05-13 09:38:42,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:42,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:42,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:42,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:42,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +26: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:42,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:42,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:42,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:42,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +26: [2023-05-13 09:38:42,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:42,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:42,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +10: [2023-05-13 09:38:42,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:42,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:42,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:42,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +10: [2023-05-13 09:38:42,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:42,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:42,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:42,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:42,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:42,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:42,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:42,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:42,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +16: [2023-05-13 09:38:42,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:42,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:42,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:42,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +13: [2023-05-13 09:38:42,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +24: [2023-05-13 09:38:42,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:42,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:42,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:42,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:42,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:42,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:42,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:42,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:42,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:42,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:42,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:42,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +10: [2023-05-13 09:38:42,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:42,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:42,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:42,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:42,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:42,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:42,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:42,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:42,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:42,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +21: [2023-05-13 09:38:42,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +21: [2023-05-13 09:38:42,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +21: [2023-05-13 09:38:42,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:42,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:42,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:42,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:42,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:42,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:42,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:42,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:42,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:42,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:42,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:42,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:42,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:42,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:42,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:42,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:42,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:42,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:42,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +15: [2023-05-13 09:38:42,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:42,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:42,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:42,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:42,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +29: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +26: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +27: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +31: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +29: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +13: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:42,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:42,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:42,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:42,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:42,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:42,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:42,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:42,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:42,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:42,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:42,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:42,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:42,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:42,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:42,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:42,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:42,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:42,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:42,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:42,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:42,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +16: [2023-05-13 09:38:42,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:42,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:42,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:42,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:42,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:42,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:42,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:42,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:42,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:42,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +16: [2023-05-13 09:38:42,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +13: [2023-05-13 09:38:42,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:42,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:42,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:42,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +17: [2023-05-13 09:38:42,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +17: [2023-05-13 09:38:42,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:42,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:42,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:42,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +13: [2023-05-13 09:38:42,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:42,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:42,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +15: [2023-05-13 09:38:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_30-model_01-model_states.pt. +24: [2023-05-13 09:38:42,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +22: [2023-05-13 09:38:42,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:42,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +21: [2023-05-13 09:38:42,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:42,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +13: [2023-05-13 09:38:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:42,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:42,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:42,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:42,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:42,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:42,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:42,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:42,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:42,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +17: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:42,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:42,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:42,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:42,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:42,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:42,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:42,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:42,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:42,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:42,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:42,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:42,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:42,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:42,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:42,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:42,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:42,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:42,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:42,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:42,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:42,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:42,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:42,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:42,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:42,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:42,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:42,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:42,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:42,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:42,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:42,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:42,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:42,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:42,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:42,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:42,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:42,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:42,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:42,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:42,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:42,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:42,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:42,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:42,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:42,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:42,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +22: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:42,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:42,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:42,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +20: [2023-05-13 09:38:42,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:42,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +17: [2023-05-13 09:38:42,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +17: [2023-05-13 09:38:42,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:42,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:42,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:42,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:42,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:42,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:42,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +22: [2023-05-13 09:38:42,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +17: [2023-05-13 09:38:42,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:42,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +18: [2023-05-13 09:38:42,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:42,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... +17: [2023-05-13 09:38:42,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +29: [2023-05-13 09:38:42,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +29: [2023-05-13 09:38:42,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:42,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:42,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:42,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +31: [2023-05-13 09:38:42,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:42,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:42,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +17: [2023-05-13 09:38:42,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:42,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:42,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:42,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:42,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:42,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:42,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +31: [2023-05-13 09:38:42,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:42,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:42,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +20: [2023-05-13 09:38:42,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +20: [2023-05-13 09:38:42,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +20: [2023-05-13 09:38:42,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +20: [2023-05-13 09:38:42,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:42,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +29: [2023-05-13 09:38:42,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +31: [2023-05-13 09:38:42,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +31: [2023-05-13 09:38:42,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:42,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +31: [2023-05-13 09:38:42,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:42,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:42,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +29: [2023-05-13 09:38:42,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:42,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:42,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +31: [2023-05-13 09:38:42,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +31: [2023-05-13 09:38:42,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +29: [2023-05-13 09:38:42,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +31: [2023-05-13 09:38:42,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +31: [2023-05-13 09:38:42,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:42,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:42,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:42,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:42,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:42,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +28: [2023-05-13 09:38:42,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:42,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:42,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:42,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +28: [2023-05-13 09:38:42,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +28: [2023-05-13 09:38:42,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:42,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:42,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:42,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:42,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:42,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +18: [2023-05-13 09:38:42,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +19: [2023-05-13 09:38:42,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +19: [2023-05-13 09:38:42,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +19: [2023-05-13 09:38:42,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +21: [2023-05-13 09:38:42,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +21: [2023-05-13 09:38:42,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +21: [2023-05-13 09:38:42,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +19: [2023-05-13 09:38:42,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:42,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +18: [2023-05-13 09:38:42,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +18: [2023-05-13 09:38:42,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +28: [2023-05-13 09:38:42,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:42,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:42,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:42,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:42,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:42,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:42,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:42,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:42,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:42,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:42,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:42,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:42,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:42,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:42,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:42,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:42,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:42,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +15: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +15: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +15: [2023-05-13 09:38:42,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:42,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:42,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:42,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:42,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:42,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:42,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:42,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:42,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:42,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:42,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:42,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:42,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:42,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:42,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:42,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:42,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:42,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:42,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:42,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:42,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:42,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:42,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:42,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:42,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:42,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:42,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +15: [2023-05-13 09:38:42,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:42,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:42,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:42,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:42,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +15: [2023-05-13 09:38:42,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:42,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +15: [2023-05-13 09:38:42,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:42,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:42,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:42,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:42,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:42,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:42,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:42,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +25: [2023-05-13 09:38:42,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +27: [2023-05-13 09:38:42,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +27: [2023-05-13 09:38:42,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +27: [2023-05-13 09:38:42,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +27: [2023-05-13 09:38:42,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:42,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:42,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:42,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:42,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +24: [2023-05-13 09:38:42,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +24: [2023-05-13 09:38:42,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:42,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +24: [2023-05-13 09:38:42,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +24: [2023-05-13 09:38:42,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:42,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:42,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:42,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:42,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:42,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:42,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:42,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:42,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:42,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +23: [2023-05-13 09:38:42,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +23: [2023-05-13 09:38:42,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:42,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:42,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:42,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:42,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:42,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:42,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:42,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +15: [2023-05-13 09:38:42,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:42,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:42,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:42,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:42,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:42,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:42,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:42,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:42,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:42,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:42,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +15: [2023-05-13 09:38:42,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:42,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:42,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:42,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:42,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:42,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:42,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +15: [2023-05-13 09:38:42,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:42,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:42,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:42,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:42,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:42,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:42,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:42,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:42,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:42,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:42,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:42,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:42,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:42,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:42,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:42,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +27: [2023-05-13 09:38:42,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +27: [2023-05-13 09:38:42,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:42,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:42,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +27: [2023-05-13 09:38:42,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:42,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:42,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:42,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:42,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:42,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:42,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:42,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +13: [2023-05-13 09:38:42,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +13: [2023-05-13 09:38:42,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:42,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:42,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:42,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:42,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:42,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:42,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +10: [2023-05-13 09:38:42,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:42,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +10: [2023-05-13 09:38:42,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:42,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:42,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +17: [2023-05-13 09:38:42,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +17: [2023-05-13 09:38:42,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +26: [2023-05-13 09:38:42,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +26: [2023-05-13 09:38:42,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +17: [2023-05-13 09:38:42,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:42,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +26: [2023-05-13 09:38:42,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:42,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:42,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:42,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:42,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:42,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:42,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:42,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:42,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:42,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:42,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:42,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:42,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:42,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:42,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:42,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:42,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:42,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:42,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:42,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:42,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:42,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:42,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:42,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:42,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:42,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:42,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:42,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:42,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:42,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:42,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:42,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:42,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:42,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:42,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +26: [2023-05-13 09:38:42,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:42,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:42,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +10: [2023-05-13 09:38:42,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:42,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:42,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:42,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +10: [2023-05-13 09:38:42,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:42,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:42,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:42,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:42,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:42,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:42,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:42,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +26: [2023-05-13 09:38:42,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +26: [2023-05-13 09:38:42,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:42,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:42,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:42,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:42,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:42,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:42,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:42,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:42,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:42,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:42,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:42,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:42,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:42,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:42,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:42,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:42,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:42,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:42,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:42,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:42,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:42,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:42,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:42,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:42,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:42,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:42,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:42,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:42,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:42,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:42,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:42,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:42,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:42,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:42,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:42,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:42,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:42,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:42,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:42,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:42,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:42,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:42,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:42,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:42,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:42,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:42,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:42,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:42,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:42,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:42,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:42,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:42,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:42,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:42,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:42,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:42,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:42,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:42,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:42,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:42,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:42,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:42,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:42,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:42,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:42,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:42,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:42,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:42,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:42,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:42,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:42,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:42,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:42,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:42,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:42,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:42,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:42,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:42,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:42,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:42,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:42,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:42,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:42,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:42,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:42,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:42,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:42,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:42,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:42,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:42,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:42,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:42,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:42,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:42,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:42,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:42,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:42,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:42,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:42,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:42,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:42,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:42,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:42,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:42,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +28: [2023-05-13 09:38:42,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:42,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:42,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +31: [2023-05-13 09:38:42,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:42,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +31: [2023-05-13 09:38:42,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +28: [2023-05-13 09:38:42,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:42,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:42,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:42,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:42,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:42,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:42,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:42,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:42,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:42,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:42,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:42,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:42,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:42,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:42,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:42,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:42,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:42,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:42,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:42,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:42,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:42,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:42,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:42,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:42,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:42,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:42,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:42,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:42,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:42,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:42,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +23: [2023-05-13 09:38:42,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:42,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:42,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:42,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +22: [2023-05-13 09:38:42,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +22: [2023-05-13 09:38:42,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +22: [2023-05-13 09:38:42,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:42,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:42,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:42,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:42,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:42,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:42,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:42,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:42,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:42,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:42,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +31: [2023-05-13 09:38:42,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +25: [2023-05-13 09:38:42,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:42,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:42,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:42,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:42,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:42,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:42,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:42,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:42,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:42,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:42,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:42,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:42,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:42,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +21: [2023-05-13 09:38:42,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:42,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:42,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:42,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:42,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:42,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:42,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:42,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:42,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:42,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:42,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:42,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +10: [2023-05-13 09:38:42,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:42,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:42,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:42,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:42,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:42,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +29: [2023-05-13 09:38:42,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:42,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +18: [2023-05-13 09:38:42,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +18: [2023-05-13 09:38:42,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:42,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:42,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +18: [2023-05-13 09:38:42,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +18: [2023-05-13 09:38:42,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +20: [2023-05-13 09:38:42,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +20: [2023-05-13 09:38:42,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +20: [2023-05-13 09:38:42,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +20: [2023-05-13 09:38:42,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:42,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:42,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:42,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:42,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:42,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +29: [2023-05-13 09:38:42,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +30: [2023-05-13 09:38:42,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:42,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:42,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:42,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:42,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:42,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +18: [2023-05-13 09:38:42,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +30: [2023-05-13 09:38:42,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +30: [2023-05-13 09:38:42,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +17: [2023-05-13 09:38:42,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:42,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:42,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:42,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:42,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:42,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +29: [2023-05-13 09:38:42,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:42,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:42,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +29: [2023-05-13 09:38:42,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:42,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:42,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:42,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:42,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:42,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:42,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +29: [2023-05-13 09:38:42,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +18: [2023-05-13 09:38:42,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:42,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:42,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:42,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +23: [2023-05-13 09:38:42,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:42,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:42,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:42,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:42,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:43,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:43,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:43,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:43,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +17: [2023-05-13 09:38:43,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:43,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +15: [2023-05-13 09:38:43,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +15: [2023-05-13 09:38:43,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +19: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +27: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +27: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +19: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +19: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +27: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +19: [2023-05-13 09:38:43,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +27: [2023-05-13 09:38:43,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:43,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +17: [2023-05-13 09:38:43,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:43,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:43,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:43,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:43,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:43,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +26: [2023-05-13 09:38:43,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +26: [2023-05-13 09:38:43,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +17: [2023-05-13 09:38:43,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +10: [2023-05-13 09:38:43,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +10: [2023-05-13 09:38:43,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:43,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +10: [2023-05-13 09:38:43,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +10: [2023-05-13 09:38:43,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +26: [2023-05-13 09:38:43,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +25: [2023-05-13 09:38:43,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +25: [2023-05-13 09:38:43,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +25: [2023-05-13 09:38:43,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +25: [2023-05-13 09:38:43,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:43,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:43,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:43,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:43,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:43,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:43,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:43,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:43,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:43,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +15: [2023-05-13 09:38:43,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:43,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:43,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:43,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:43,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:43,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:43,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:43,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:43,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:43,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:43,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:43,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +10: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:43,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:43,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:43,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:43,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:43,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:43,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:43,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:43,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +23: [2023-05-13 09:38:43,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:43,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:43,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:43,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +12: [2023-05-13 09:38:43,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:43,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:43,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +25: [2023-05-13 09:38:43,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:43,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:43,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:43,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:43,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:43,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:43,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:43,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:43,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:43,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:43,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:43,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:43,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:43,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:43,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:43,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:43,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:43,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:43,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:43,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:43,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:43,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:43,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:43,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:43,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:43,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:43,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:43,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:43,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:43,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:43,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +25: [2023-05-13 09:38:43,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +19: [2023-05-13 09:38:43,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:43,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:43,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:43,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:43,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +19: [2023-05-13 09:38:43,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:43,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:43,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:43,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:43,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +28: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +28: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +10: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:43,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:43,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:43,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:43,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:43,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:43,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +25: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:43,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +19: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:43,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:43,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +13: [2023-05-13 09:38:43,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +13: [2023-05-13 09:38:43,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:43,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +19: [2023-05-13 09:38:43,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +19: [2023-05-13 09:38:43,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:43,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:43,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:43,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:43,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +12: [2023-05-13 09:38:43,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:43,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:43,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:43,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:43,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:43,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +10: [2023-05-13 09:38:43,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +10: [2023-05-13 09:38:43,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +18: [2023-05-13 09:38:43,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:43,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:43,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:43,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:43,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:43,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:43,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:43,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:43,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:43,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:43,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:43,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:43,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +12: [2023-05-13 09:38:43,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:43,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:43,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:43,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:43,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +18: [2023-05-13 09:38:43,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:43,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:43,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:43,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:43,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:43,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:43,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +11: [2023-05-13 09:38:43,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:43,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:43,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:43,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:43,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:43,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:43,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:43,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:43,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:43,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:43,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:43,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:43,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:43,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:43,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:43,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:43,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:43,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:43,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:43,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:43,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:43,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:43,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:43,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:43,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:43,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:43,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:43,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:43,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:43,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +28: [2023-05-13 09:38:43,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:43,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:43,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:43,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:43,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:43,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:43,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:43,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:43,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:43,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +28: [2023-05-13 09:38:43,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:43,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:43,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:43,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:43,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:43,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:43,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:43,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:43,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:43,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:43,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:43,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:43,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:43,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:43,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +21: [2023-05-13 09:38:43,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +21: [2023-05-13 09:38:43,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +11: [2023-05-13 09:38:43,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:43,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:43,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:43,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:43,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:43,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:43,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:43,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:43,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:43,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:43,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:43,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:43,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:43,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:43,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:43,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:43,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +10: [2023-05-13 09:38:43,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:43,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:43,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:43,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:43,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:43,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:43,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:43,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:43,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:43,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:43,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:43,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:43,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:43,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:43,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:43,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:43,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:43,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:43,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:43,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:43,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:43,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:43,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:43,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:43,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:43,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:43,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:43,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:43,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:43,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:43,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:43,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:43,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:43,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:43,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:43,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:43,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:43,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:43,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:43,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:43,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:43,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:43,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:43,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:43,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:43,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:43,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:43,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:43,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:43,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:43,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:43,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:43,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:43,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:43,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:43,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:43,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:43,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:43,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:43,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:43,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:43,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:43,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:43,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:43,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:43,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:43,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:43,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:43,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:43,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:43,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:43,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:43,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:43,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:43,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:43,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:43,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:43,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:43,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:43,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:43,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:43,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:43,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:43,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:43,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:43,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:43,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:43,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:43,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:43,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:43,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +30: [2023-05-13 09:38:43,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:43,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:43,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:43,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt... +17: [2023-05-13 09:38:43,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:43,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:43,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:43,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:43,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:43,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:43,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:43,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:43,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:43,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:43,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:43,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:43,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:43,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:43,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:43,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:43,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:43,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:43,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:43,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:43,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:43,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:43,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:43,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:43,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:43,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:43,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:43,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:43,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:43,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:43,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:43,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:43,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:43,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:43,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:43,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:43,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:43,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:43,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:43,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:43,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:43,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:43,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:43,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:43,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:43,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:43,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:43,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:43,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:43,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:43,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:43,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:43,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:43,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:43,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:43,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:43,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:43,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:43,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:43,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:43,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:43,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:43,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:43,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:43,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:43,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:43,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:43,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:43,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:43,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:43,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:43,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:43,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:43,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:43,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:43,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:43,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:43,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:43,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:43,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:43,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:43,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:43,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:43,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:43,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:43,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:43,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:43,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:43,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:43,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:43,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:43,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:43,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:43,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +11: [2023-05-13 09:38:43,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:43,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:43,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:43,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +20: [2023-05-13 09:38:43,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +20: [2023-05-13 09:38:43,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +20: [2023-05-13 09:38:43,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:43,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:43,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:43,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:43,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +20: [2023-05-13 09:38:43,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:43,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:43,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:43,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +20: [2023-05-13 09:38:43,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +20: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:43,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +29: [2023-05-13 09:38:43,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +29: [2023-05-13 09:38:43,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +29: [2023-05-13 09:38:43,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +29: [2023-05-13 09:38:43,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:43,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:43,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:43,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +31: [2023-05-13 09:38:43,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:43,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:43,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:43,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:43,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:43,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:43,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +15: [2023-05-13 09:38:43,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +15: [2023-05-13 09:38:43,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +15: [2023-05-13 09:38:43,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:43,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:43,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:43,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:43,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:43,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:43,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:43,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:43,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:43,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:43,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:43,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +22: [2023-05-13 09:38:43,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:43,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:43,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:43,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:43,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:43,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +14: [2023-05-13 09:38:43,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +14: [2023-05-13 09:38:43,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:43,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:43,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:43,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:43,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:43,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +14: [2023-05-13 09:38:43,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +14: [2023-05-13 09:38:43,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +18: [2023-05-13 09:38:43,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:43,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:43,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:43,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:43,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +17: [2023-05-13 09:38:43,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +17: [2023-05-13 09:38:43,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +17: [2023-05-13 09:38:43,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:43,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:43,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:43,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +29: [2023-05-13 09:38:43,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +18: [2023-05-13 09:38:43,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:43,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:43,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:43,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:43,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:43,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:43,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:43,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:43,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:43,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:43,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +22: [2023-05-13 09:38:43,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:43,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:43,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +22: [2023-05-13 09:38:43,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +22: [2023-05-13 09:38:43,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:43,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:43,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:43,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:43,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:43,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:43,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +15: [2023-05-13 09:38:43,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:43,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +18: [2023-05-13 09:38:43,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:43,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:43,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:43,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:43,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:43,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:43,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:43,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:43,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +18: [2023-05-13 09:38:43,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +18: [2023-05-13 09:38:43,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +17: [2023-05-13 09:38:43,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:43,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +30: [2023-05-13 09:38:43,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:43,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +24: [2023-05-13 09:38:43,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +24: [2023-05-13 09:38:43,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +24: [2023-05-13 09:38:43,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:43,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +24: [2023-05-13 09:38:43,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:43,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:43,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:43,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:43,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +17: [2023-05-13 09:38:43,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:43,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +17: [2023-05-13 09:38:43,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +24: [2023-05-13 09:38:43,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:43,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:43,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:43,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:43,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:43,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:43,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +30: [2023-05-13 09:38:43,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:43,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +13: [2023-05-13 09:38:43,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +13: [2023-05-13 09:38:43,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +13: [2023-05-13 09:38:43,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +24: [2023-05-13 09:38:43,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:43,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +30: [2023-05-13 09:38:43,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:43,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:43,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:43,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +26: [2023-05-13 09:38:43,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +12: [2023-05-13 09:38:43,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:43,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:43,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:43,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:43,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +30: [2023-05-13 09:38:43,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +26: [2023-05-13 09:38:43,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +12: [2023-05-13 09:38:43,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:43,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:43,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:43,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:43,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:43,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:43,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:43,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:43,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:43,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:43,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:43,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +13: [2023-05-13 09:38:43,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:43,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:43,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:43,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +24: [2023-05-13 09:38:43,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:43,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:43,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +24: [2023-05-13 09:38:43,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +10: [2023-05-13 09:38:43,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:43,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:43,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:43,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:43,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:43,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +26: [2023-05-13 09:38:43,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:43,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:43,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:43,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:43,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:43,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:43,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:43,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:43,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:43,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:43,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:43,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:43,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:43,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:43,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:43,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:43,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 2: [2023-05-13 09:38:43,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:43,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:43,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 2: [2023-05-13 09:38:43,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:43,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:43,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:43,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:43,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:43,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:43,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:43,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:43,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:43,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:43,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:43,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:43,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:43,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:43,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:43,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:43,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +28: [2023-05-13 09:38:43,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:43,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +26: [2023-05-13 09:38:43,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:43,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +26: [2023-05-13 09:38:43,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:43,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +18: [2023-05-13 09:38:43,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:43,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:43,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:43,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:43,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:43,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:43,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:43,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:43,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +10: [2023-05-13 09:38:43,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:43,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +27: [2023-05-13 09:38:43,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +27: [2023-05-13 09:38:43,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +10: [2023-05-13 09:38:43,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:43,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:43,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +12: [2023-05-13 09:38:43,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +12: [2023-05-13 09:38:43,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +12: [2023-05-13 09:38:43,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:43,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +10: [2023-05-13 09:38:43,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +27: [2023-05-13 09:38:43,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:43,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:43,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:43,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:43,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:43,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +28: [2023-05-13 09:38:43,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:43,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:43,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +16: [2023-05-13 09:38:43,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:43,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 2: [2023-05-13 09:38:43,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:43,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:43,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +23: [2023-05-13 09:38:43,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +23: [2023-05-13 09:38:43,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:43,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:43,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +23: [2023-05-13 09:38:43,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:43,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:43,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +28: [2023-05-13 09:38:43,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:43,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:43,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:43,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:43,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:43,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:43,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:43,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:43,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:43,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:43,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:43,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:43,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:43,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:43,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:43,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:43,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:43,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:43,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:43,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:43,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:43,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +25: [2023-05-13 09:38:43,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +25: [2023-05-13 09:38:43,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +25: [2023-05-13 09:38:43,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +21: [2023-05-13 09:38:43,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +21: [2023-05-13 09:38:43,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +21: [2023-05-13 09:38:43,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:43,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:43,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:43,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +25: [2023-05-13 09:38:43,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:43,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:43,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:43,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:43,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:43,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:43,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:43,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:43,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:43,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:43,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:43,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:43,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:43,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:43,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +16: [2023-05-13 09:38:43,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:43,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +16: [2023-05-13 09:38:43,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +16: [2023-05-13 09:38:43,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:43,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:43,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:43,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:43,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +23: [2023-05-13 09:38:43,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:43,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:43,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:43,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:43,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:43,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +23: [2023-05-13 09:38:43,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:43,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:43,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:43,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:43,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:43,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:43,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:43,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:43,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:43,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:43,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:43,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:43,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:43,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:43,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:43,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:43,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:43,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:43,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:43,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +23: [2023-05-13 09:38:43,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:43,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:43,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:43,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:43,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:43,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:43,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:43,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:43,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +11: [2023-05-13 09:38:43,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. +20: [2023-05-13 09:38:43,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_33-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:43,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:43,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:43,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:43,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:43,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:43,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:43,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:43,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:43,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:43,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:43,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:43,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:43,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:43,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:43,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:43,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +25: [2023-05-13 09:38:43,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:43,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +11: [2023-05-13 09:38:43,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:43,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:43,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:43,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:43,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:43,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:43,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:43,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:43,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:43,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:43,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:43,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:43,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:43,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:43,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:43,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:43,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:43,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:43,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:43,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:43,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:43,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:43,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:43,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:43,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:43,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:43,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:43,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:43,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:43,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:43,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +25: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... +11: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:43,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:43,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:43,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:43,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:43,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:43,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:43,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:43,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:43,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:43,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:43,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:43,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:43,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:43,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:43,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:43,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:43,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:43,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:43,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:43,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:43,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:43,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:43,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:43,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:43,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:43,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:43,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:43,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:43,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:43,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:43,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:43,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:43,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:43,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:43,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:43,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:43,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:43,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:43,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:43,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:43,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:43,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:43,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:43,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:43,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:43,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:43,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:43,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:43,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:43,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:43,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:43,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:43,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:43,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:43,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:43,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:43,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:43,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:43,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:43,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:43,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:43,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:43,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:43,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:43,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:43,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:43,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:43,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:43,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:43,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:43,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:43,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:43,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:43,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:43,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:43,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:43,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:43,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:43,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:43,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:43,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:43,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:43,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:43,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:43,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:43,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:43,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:43,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:43,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:43,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:43,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:43,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:43,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:43,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:43,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:43,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:43,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:43,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:43,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:43,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:43,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:43,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:43,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:43,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:43,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:43,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:43,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:43,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:43,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:43,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:43,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:43,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:43,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:43,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:43,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:43,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:43,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:43,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:43,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:43,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:43,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:43,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:43,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:43,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:43,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:43,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:43,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:43,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:43,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:43,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:43,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:43,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:43,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:43,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:43,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:43,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:43,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:43,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:43,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:43,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:43,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:43,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:43,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:43,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:43,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:43,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:43,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:43,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:43,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:43,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:43,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:43,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:43,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:43,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:43,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:43,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:43,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:43,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:43,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:43,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:43,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:43,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:43,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:43,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:43,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:43,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:43,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:43,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:43,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:43,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:43,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:43,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:43,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:43,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:43,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:43,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:43,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:43,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:43,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:43,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:43,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:43,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:43,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:43,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:43,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:43,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:43,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:43,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:43,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:43,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:43,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:43,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:43,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:43,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:43,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:43,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:43,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:43,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:43,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:43,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:43,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:43,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:43,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:43,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:43,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:43,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:43,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:43,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:43,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:43,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:43,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:43,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:43,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:43,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:43,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:43,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:43,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:43,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:44,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:44,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:44,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:44,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:44,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:44,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:44,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:44,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:44,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:44,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:44,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:44,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:44,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:44,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:44,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:44,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:44,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:44,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:44,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:44,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:44,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:44,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:44,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:44,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:44,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:44,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:44,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:44,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:44,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:44,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:44,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:44,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:44,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:44,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:44,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:44,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:44,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:44,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:44,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:44,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:44,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:44,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:44,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:44,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:44,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:44,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:44,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:44,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:44,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:44,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:44,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:44,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:44,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:44,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:44,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:44,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:44,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:44,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:44,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:44,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:44,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:44,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:44,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:44,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:44,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:44,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:44,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:44,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:44,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:44,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:44,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:44,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:44,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:44,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:44,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:44,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:44,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:44,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:44,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:44,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:44,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:44,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:44,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:44,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:44,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:44,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:44,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:44,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:44,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:44,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:44,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:44,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:44,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:44,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:44,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:44,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:44,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:44,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:44,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:44,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:44,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:44,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:44,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:44,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:44,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:44,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:44,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:44,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:44,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:44,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:44,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:44,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:44,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:44,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:44,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:44,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +22: [2023-05-13 09:38:44,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +15: [2023-05-13 09:38:44,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:44,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:44,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:44,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:44,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:44,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:44,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:44,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:44,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:44,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:44,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:44,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:44,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:44,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:44,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:44,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:44,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:44,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:44,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:44,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:44,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:44,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:44,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:44,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:44,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:44,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:44,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:44,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:44,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:44,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:44,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:44,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:44,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:44,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:44,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:44,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:44,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:44,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:44,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:44,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:44,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:44,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:44,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:44,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:44,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:44,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:44,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:44,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:44,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:44,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:44,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:44,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:44,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:44,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:44,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:44,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:44,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:44,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:44,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:44,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:44,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:44,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:44,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:44,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:44,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:44,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:44,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:44,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:44,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:44,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:44,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:44,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:44,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:44,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:44,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:44,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:44,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:44,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:44,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:44,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:44,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:44,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:44,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:44,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:44,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:44,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:44,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:44,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +28: [2023-05-13 09:38:44,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:44,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +28: [2023-05-13 09:38:44,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +19: [2023-05-13 09:38:44,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:44,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:44,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:44,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:44,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +28: [2023-05-13 09:38:44,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:44,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:44,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:44,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:44,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +19: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:44,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:44,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:44,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:44,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:44,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:44,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:44,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:44,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:44,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:44,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:44,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:44,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:44,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:44,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +28: [2023-05-13 09:38:44,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:44,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:44,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:44,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:44,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:44,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 4: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +28: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:44,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:44,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +26: [2023-05-13 09:38:44,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +26: [2023-05-13 09:38:44,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +26: [2023-05-13 09:38:44,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:44,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +26: [2023-05-13 09:38:44,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:44,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +20: [2023-05-13 09:38:44,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:44,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:44,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +28: [2023-05-13 09:38:44,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:44,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:44,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +16: [2023-05-13 09:38:44,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +16: [2023-05-13 09:38:44,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +16: [2023-05-13 09:38:44,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +28: [2023-05-13 09:38:44,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:44,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:44,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:44,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:44,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:44,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:44,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:44,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 4: [2023-05-13 09:38:44,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:44,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:44,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +20: [2023-05-13 09:38:44,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:44,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:44,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:44,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:44,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:44,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:44,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:44,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:44,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:44,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:44,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:44,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:44,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:44,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +18: [2023-05-13 09:38:44,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +20: [2023-05-13 09:38:44,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:44,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:44,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:44,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:44,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:44,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +13: [2023-05-13 09:38:44,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +13: [2023-05-13 09:38:44,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:44,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +20: [2023-05-13 09:38:44,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:44,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:44,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:44,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:44,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:44,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:44,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:44,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:44,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:44,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:44,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:44,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:44,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:44,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:44,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:44,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +18: [2023-05-13 09:38:44,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:44,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 4: [2023-05-13 09:38:44,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:44,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:44,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 4: [2023-05-13 09:38:44,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +21: [2023-05-13 09:38:44,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +21: [2023-05-13 09:38:44,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +21: [2023-05-13 09:38:44,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +21: [2023-05-13 09:38:44,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +13: [2023-05-13 09:38:44,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:44,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:44,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:44,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:44,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:44,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:44,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:44,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:44,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:44,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:44,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:44,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +19: [2023-05-13 09:38:44,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +19: [2023-05-13 09:38:44,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +19: [2023-05-13 09:38:44,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:44,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:44,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:44,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:44,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:44,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:44,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:44,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:44,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:44,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:44,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:44,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:44,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:44,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:44,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:44,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:44,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:44,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:44,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:44,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:44,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:44,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:44,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +21: [2023-05-13 09:38:44,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:44,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:44,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:44,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:44,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +21: [2023-05-13 09:38:44,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:44,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 5: [2023-05-13 09:38:44,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:44,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:44,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:44,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:44,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:44,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:44,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:44,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:44,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 5: [2023-05-13 09:38:44,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:44,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:44,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:44,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:44,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:44,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:44,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:44,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:44,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:44,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:44,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:44,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +10: [2023-05-13 09:38:44,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:44,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:44,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:44,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:44,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +10: [2023-05-13 09:38:44,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:44,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:44,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 6: [2023-05-13 09:38:44,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:44,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:44,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 6: [2023-05-13 09:38:44,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:44,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:44,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 6: [2023-05-13 09:38:44,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:44,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:44,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:44,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:44,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:44,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:44,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:44,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:44,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:44,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:44,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:44,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:44,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:44,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:44,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:44,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:44,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:44,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:44,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:44,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:44,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:44,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:44,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:44,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:44,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:44,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:44,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:44,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:44,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:44,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:44,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:44,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:44,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:44,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:44,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:44,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:44,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:44,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:44,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:44,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:44,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:44,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:44,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:44,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:44,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:44,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:44,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:44,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:44,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:44,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:44,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:44,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:44,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:44,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:44,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:44,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:44,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:44,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:44,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:44,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:44,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:44,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:44,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:44,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:44,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:44,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:44,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:44,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:44,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:44,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:44,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:44,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +22: [2023-05-13 09:38:44,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +22: [2023-05-13 09:38:44,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:44,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:44,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:44,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +25: [2023-05-13 09:38:44,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:44,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +23: [2023-05-13 09:38:44,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +22: [2023-05-13 09:38:44,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +23: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:44,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:44,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:44,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:44,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 9: [2023-05-13 09:38:44,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +20: [2023-05-13 09:38:44,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +15: [2023-05-13 09:38:44,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:44,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +17: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +18: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +15: [2023-05-13 09:38:44,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +18: [2023-05-13 09:38:44,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +18: [2023-05-13 09:38:44,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:44,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:44,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:44,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:44,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +23: [2023-05-13 09:38:44,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +23: [2023-05-13 09:38:44,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +17: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +12: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +12: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:44,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +30: [2023-05-13 09:38:44,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:44,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:44,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:44,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:44,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:44,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +30: [2023-05-13 09:38:44,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:44,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:44,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:44,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:44,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:44,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +27: [2023-05-13 09:38:44,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +27: [2023-05-13 09:38:44,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:44,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 5: [2023-05-13 09:38:44,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:44,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:44,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:44,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 5: [2023-05-13 09:38:44,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:44,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:44,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:44,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +25: [2023-05-13 09:38:44,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:44,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:44,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +10: [2023-05-13 09:38:44,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:44,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:44,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:44,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:44,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +27: [2023-05-13 09:38:44,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 5: [2023-05-13 09:38:44,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:44,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:44,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:44,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:44,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:44,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:44,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:44,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:44,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:44,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:44,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:44,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:44,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:44,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:44,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:44,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:44,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 5: [2023-05-13 09:38:44,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:44,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:44,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:44,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:44,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +26: [2023-05-13 09:38:44,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:44,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +30: [2023-05-13 09:38:44,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:44,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +23: [2023-05-13 09:38:44,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:44,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:44,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:44,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:44,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:44,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:44,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:44,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:44,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:44,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:44,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:44,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:44,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:44,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:44,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:44,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:44,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:44,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:44,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:44,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:44,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:44,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:44,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:44,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:44,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:44,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:44,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:44,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:44,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:44,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:44,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:44,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:44,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:44,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:44,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:44,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:44,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:44,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:44,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +23: [2023-05-13 09:38:44,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:44,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +26: [2023-05-13 09:38:44,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:44,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:44,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:44,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +25: [2023-05-13 09:38:44,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +18: [2023-05-13 09:38:44,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:44,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:44,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:44,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:44,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:44,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:44,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +22: [2023-05-13 09:38:44,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:44,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:44,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:44,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:44,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:44,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:44,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:44,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:44,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:44,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:44,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:44,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:44,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:44,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:44,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:44,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:44,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:44,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:44,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +30: [2023-05-13 09:38:44,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:44,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:44,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +22: [2023-05-13 09:38:44,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:44,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:44,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:44,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:44,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:44,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:44,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:44,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:44,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:44,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:44,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:44,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:44,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:44,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:44,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:44,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +23: [2023-05-13 09:38:44,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:44,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:44,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:44,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:44,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 7: [2023-05-13 09:38:44,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:44,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:44,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:44,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +23: [2023-05-13 09:38:44,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:44,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:44,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:44,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:44,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:44,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +30: [2023-05-13 09:38:44,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:44,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +22: [2023-05-13 09:38:44,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:44,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:44,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +30: [2023-05-13 09:38:44,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +22: [2023-05-13 09:38:44,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:44,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +13: [2023-05-13 09:38:44,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:44,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:44,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:44,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:44,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:44,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:44,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:44,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:44,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:44,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:44,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:44,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:44,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:44,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:44,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:44,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +23: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +16: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +13: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:44,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:44,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:44,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:44,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:44,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:44,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:44,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:44,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:44,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:44,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:44,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:44,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:44,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:44,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:44,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:44,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:44,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:44,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:44,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:44,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:44,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:44,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:44,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:44,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +13: [2023-05-13 09:38:44,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:44,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:44,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:44,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:44,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:44,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:44,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:44,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:44,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +20: [2023-05-13 09:38:44,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:44,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:44,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:44,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:44,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:44,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:44,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:44,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:44,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:44,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:44,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:44,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +19: [2023-05-13 09:38:44,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:44,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:44,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:44,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:44,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:44,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:44,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:44,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:44,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:44,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:44,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:44,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +28: [2023-05-13 09:38:44,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 6: [2023-05-13 09:38:44,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:44,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:44,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:44,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:44,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:44,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:44,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:44,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:44,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:44,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:44,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:44,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:44,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:44,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:44,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:44,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:44,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:44,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:44,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:44,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:44,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:44,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:44,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:44,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:44,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:44,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:44,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:44,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:44,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:44,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:44,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:44,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:44,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:44,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:44,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:44,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:44,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:44,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:44,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:44,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:44,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:44,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:44,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:44,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:44,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:44,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:44,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:44,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:44,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:44,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:44,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:44,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:44,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:44,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:44,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:44,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:44,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:44,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:44,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:44,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:44,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:44,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:44,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:44,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:44,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:44,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:44,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:44,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:44,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:44,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:44,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:44,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:44,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:44,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:44,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:44,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:44,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:44,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:44,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:44,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:44,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:44,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:44,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:44,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:44,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:44,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:44,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:44,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:44,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:44,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:44,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:44,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:44,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:44,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:44,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:44,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:44,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:44,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:44,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:44,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:44,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:44,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:44,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:44,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:44,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:44,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:44,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:44,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:44,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:44,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:44,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:44,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:44,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +15: [2023-05-13 09:38:44,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:44,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:44,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:44,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:44,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:44,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:44,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:44,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:44,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:44,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:44,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:44,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:44,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:44,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:44,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:44,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 5: [2023-05-13 09:38:44,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:44,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:44,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:44,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:44,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:44,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +14: [2023-05-13 09:38:44,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +14: [2023-05-13 09:38:44,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +14: [2023-05-13 09:38:44,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +17: [2023-05-13 09:38:44,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:44,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:44,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:44,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:44,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:44,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:44,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:44,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:44,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:44,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:44,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:44,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:44,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:44,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:44,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:44,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:44,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:44,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:44,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:44,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:44,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:44,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:44,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:44,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:44,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +12: [2023-05-13 09:38:44,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:44,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +17: [2023-05-13 09:38:44,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:44,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:44,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:44,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:44,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:38:44,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:44,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:44,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:44,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:44,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:44,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +21: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +21: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +12: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +12: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:44,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:44,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:44,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:44,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:44,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:44,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:44,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:44,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:44,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:44,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... + 5: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... + 5: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... + 5: [2023-05-13 09:38:44,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:44,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +29: [2023-05-13 09:38:44,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +29: [2023-05-13 09:38:44,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +29: [2023-05-13 09:38:44,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +13: [2023-05-13 09:38:44,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:44,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +13: [2023-05-13 09:38:44,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:44,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:44,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:44,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:44,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:44,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:44,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:44,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:44,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:44,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:44,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:44,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:44,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:44,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:44,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:44,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +12: [2023-05-13 09:38:44,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +12: [2023-05-13 09:38:44,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +12: [2023-05-13 09:38:44,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:38:44,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:44,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:44,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:44,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:44,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:38:44,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:44,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +21: [2023-05-13 09:38:44,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +21: [2023-05-13 09:38:44,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +21: [2023-05-13 09:38:44,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:44,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:44,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:44,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:44,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:44,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:44,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +21: [2023-05-13 09:38:44,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:44,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:44,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:44,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +16: [2023-05-13 09:38:44,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:38:44,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:44,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:44,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:44,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +16: [2023-05-13 09:38:44,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:44,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:44,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:44,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:44,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:44,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt... +20: [2023-05-13 09:38:44,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:38:44,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt... +20: [2023-05-13 09:38:44,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt... +18: [2023-05-13 09:38:44,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:44,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:44,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:44,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:44,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:44,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:44,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:38:44,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:44,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:44,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +31: [2023-05-13 09:38:44,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:44,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +31: [2023-05-13 09:38:44,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:44,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:44,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:44,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:44,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:44,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +19: [2023-05-13 09:38:44,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +19: [2023-05-13 09:38:44,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +19: [2023-05-13 09:38:44,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +31: [2023-05-13 09:38:44,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:44,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:44,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:44,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:44,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:44,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:44,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:44,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:44,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:44,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:44,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:44,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:44,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:44,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:44,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:44,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +24: [2023-05-13 09:38:44,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:44,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:44,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:44,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:38:44,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:44,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:44,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:44,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:44,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:44,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:44,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:44,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:44,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:44,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:44,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:44,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:44,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:44,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:44,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:44,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:44,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:44,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:44,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:44,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:44,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:44,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:44,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:44,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:44,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:44,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:44,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:44,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:44,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:44,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:38:44,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 4: [2023-05-13 09:38:44,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +11: [2023-05-13 09:38:44,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:44,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:44,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +25: [2023-05-13 09:38:44,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:44,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:44,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:44,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:44,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +25: [2023-05-13 09:38:44,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +16: [2023-05-13 09:38:44,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +20: [2023-05-13 09:38:44,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +20: [2023-05-13 09:38:44,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +20: [2023-05-13 09:38:44,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 2: [2023-05-13 09:38:44,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +27: [2023-05-13 09:38:44,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +27: [2023-05-13 09:38:44,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:44,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +27: [2023-05-13 09:38:44,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:44,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:44,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:44,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:44,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:44,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:44,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:44,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:44,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:44,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:44,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:44,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:44,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:44,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:44,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +16: [2023-05-13 09:38:44,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +29: [2023-05-13 09:38:44,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:44,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:44,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:44,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:44,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:44,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:44,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:44,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +29: [2023-05-13 09:38:44,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:44,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:44,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:44,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:44,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +23: [2023-05-13 09:38:44,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:38:44,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:44,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:44,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:44,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:44,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:44,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:44,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:44,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:44,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +23: [2023-05-13 09:38:44,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:44,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:44,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:44,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:44,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:44,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt... +18: [2023-05-13 09:38:44,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt... +18: [2023-05-13 09:38:44,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt... +18: [2023-05-13 09:38:44,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:38:44,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:44,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:44,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:44,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:44,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +17: [2023-05-13 09:38:44,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +17: [2023-05-13 09:38:44,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:44,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:44,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +19: [2023-05-13 09:38:44,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +17: [2023-05-13 09:38:44,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +26: [2023-05-13 09:38:44,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +26: [2023-05-13 09:38:44,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +26: [2023-05-13 09:38:44,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +26: [2023-05-13 09:38:44,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +17: [2023-05-13 09:38:44,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... + 3: [2023-05-13 09:38:44,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:44,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... + 3: [2023-05-13 09:38:44,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:44,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:44,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:44,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:44,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:38:44,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:38:44,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:38:44,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:44,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:44,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +19: [2023-05-13 09:38:44,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:44,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +17: [2023-05-13 09:38:44,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:44,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:38:44,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:38:44,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:38:44,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:38:44,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:44,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:44,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:44,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:44,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:44,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:44,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:44,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +29: [2023-05-13 09:38:44,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:44,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:44,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:44,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:44,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:38:44,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:38:44,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:38:44,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... +20: [2023-05-13 09:38:44,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:44,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:44,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt... +24: [2023-05-13 09:38:44,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt... +24: [2023-05-13 09:38:44,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt... +24: [2023-05-13 09:38:44,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt... +20: [2023-05-13 09:38:44,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:44,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:44,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:44,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:44,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:44,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:44,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:44,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:44,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:38:44,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:38:44,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +27: [2023-05-13 09:38:44,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:44,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:44,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +27: [2023-05-13 09:38:44,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:44,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:44,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:44,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +29: [2023-05-13 09:38:44,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:44,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:44,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:44,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:44,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:44,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +29: [2023-05-13 09:38:44,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:38:44,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:38:44,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:38:44,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +23: [2023-05-13 09:38:44,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:44,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:44,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:44,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:44,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:44,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:44,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:44,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:44,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:44,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:44,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:44,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:44,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:44,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:44,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:44,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:44,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:44,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:44,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:44,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:44,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:44,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +29: [2023-05-13 09:38:44,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:44,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:44,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:44,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:38:44,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:44,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:44,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +12: [2023-05-13 09:38:44,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:44,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:44,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:44,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:44,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:44,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +28: [2023-05-13 09:38:44,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:38:44,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:38:44,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +28: [2023-05-13 09:38:44,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:38:44,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:44,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +15: [2023-05-13 09:38:44,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:44,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:44,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:44,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +12: [2023-05-13 09:38:44,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:44,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:44,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:44,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:44,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:44,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:44,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:44,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +31: [2023-05-13 09:38:44,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +31: [2023-05-13 09:38:44,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:44,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:44,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:44,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:44,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:44,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:44,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:44,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt... +23: [2023-05-13 09:38:44,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt... +23: [2023-05-13 09:38:44,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt... +23: [2023-05-13 09:38:44,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt... +24: [2023-05-13 09:38:44,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:44,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:44,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:38:44,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:44,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt... +29: [2023-05-13 09:38:44,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt... +29: [2023-05-13 09:38:44,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:38:44,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:44,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:44,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:44,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:44,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:38:44,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:44,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:44,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:44,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:44,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:44,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:44,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:44,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:44,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:44,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:44,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:44,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:44,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +13: [2023-05-13 09:38:44,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:44,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:44,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:44,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:44,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:44,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:44,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:38:44,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:44,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:44,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:44,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:44,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:44,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:44,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:44,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:44,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:44,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:38:44,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:38:44,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:38:44,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +18: [2023-05-13 09:38:44,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +18: [2023-05-13 09:38:44,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +18: [2023-05-13 09:38:44,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:44,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:44,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:44,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:44,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:44,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:44,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:44,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +26: [2023-05-13 09:38:44,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:44,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:44,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:44,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:44,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:44,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:38:44,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:44,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:44,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:44,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:44,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:44,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:44,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:44,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:44,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:44,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +15: [2023-05-13 09:38:44,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +15: [2023-05-13 09:38:44,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:45,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:45,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:45,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:45,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:45,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:45,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:45,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +31: [2023-05-13 09:38:45,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:45,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:45,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +31: [2023-05-13 09:38:45,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:45,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:38:45,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:38:45,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:38:45,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... + 4: [2023-05-13 09:38:45,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:45,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:45,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:45,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:45,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:45,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:45,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:38:45,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:45,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:38:45,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:45,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:45,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +31: [2023-05-13 09:38:45,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:45,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:45,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:45,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:45,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:45,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:45,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:45,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:45,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:45,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:45,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:45,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:45,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:45,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:45,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:45,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:45,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:45,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:45,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:45,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:45,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:45,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:45,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:45,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:45,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:45,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:45,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:45,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:45,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:45,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:45,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:45,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:45,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:45,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:45,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:45,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:45,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:45,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:45,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:45,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:45,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:45,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:45,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:45,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +16: [2023-05-13 09:38:45,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:45,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:45,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:45,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:45,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +26: [2023-05-13 09:38:45,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:45,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:45,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:45,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +16: [2023-05-13 09:38:45,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:45,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:45,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:45,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +26: [2023-05-13 09:38:45,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:45,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:45,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:45,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:45,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:45,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:45,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:45,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:45,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:45,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:45,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:45,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:45,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:45,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:45,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:45,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:45,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:45,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:45,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:45,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:45,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:45,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:45,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:45,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:45,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:45,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:45,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:38:45,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:45,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:45,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +19: [2023-05-13 09:38:45,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:45,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +14: [2023-05-13 09:38:45,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:45,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:45,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:45,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:45,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:45,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:45,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:45,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:45,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:45,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:45,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:45,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +30: [2023-05-13 09:38:45,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:45,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:45,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:45,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:45,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:45,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:45,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:45,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +21: [2023-05-13 09:38:45,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:38:45,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt... +22: [2023-05-13 09:38:45,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt... +22: [2023-05-13 09:38:45,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt... +22: [2023-05-13 09:38:45,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:38:45,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +21: [2023-05-13 09:38:45,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:45,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:45,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:45,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:45,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +27: [2023-05-13 09:38:45,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +30: [2023-05-13 09:38:45,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:45,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:45,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:45,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:45,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:45,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:45,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:38:45,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:38:45,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:38:45,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:45,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:45,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:45,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:45,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:45,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:45,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +21: [2023-05-13 09:38:45,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:45,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:38:45,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:38:45,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:38:45,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:45,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:45,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:45,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:45,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:45,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:45,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:45,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:45,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:45,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:45,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:45,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:45,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt... +26: [2023-05-13 09:38:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt... +26: [2023-05-13 09:38:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:45,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:45,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... + 4: [2023-05-13 09:38:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... + 4: [2023-05-13 09:38:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:38:45,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:45,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:45,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:45,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:45,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:45,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:45,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:45,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:45,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:45,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:45,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:45,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:45,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:45,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:45,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:45,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:45,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:45,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:45,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:45,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:45,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:45,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +22: [2023-05-13 09:38:45,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:45,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +22: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 3: [2023-05-13 09:38:45,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +29: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 3: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +29: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:45,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:45,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:45,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:45,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:45,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:45,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +11: [2023-05-13 09:38:45,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:45,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:45,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:45,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:45,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:45,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:45,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:45,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:45,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:45,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 5: [2023-05-13 09:38:45,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:45,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:45,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:45,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:45,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:45,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:45,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:45,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:45,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:45,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:45,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:45,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... + 8: [2023-05-13 09:38:45,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... + 8: [2023-05-13 09:38:45,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... + 8: [2023-05-13 09:38:45,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... +19: [2023-05-13 09:38:45,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:45,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:45,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:45,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:45,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:45,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:45,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:45,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:45,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:45,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:45,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:45,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:45,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:45,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt... +19: [2023-05-13 09:38:45,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:45,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:45,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:45,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:45,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:45,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:45,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:45,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:45,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:45,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:45,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:45,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:45,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:45,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:45,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:45,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:45,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:45,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:45,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:45,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:45,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +13: [2023-05-13 09:38:45,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +13: [2023-05-13 09:38:45,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +28: [2023-05-13 09:38:45,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:45,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +14: [2023-05-13 09:38:45,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:45,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:45,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:45,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:38:45,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:45,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +21: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +21: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:45,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:45,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:38:45,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:45,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:45,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:45,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +30: [2023-05-13 09:38:45,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:45,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:45,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:45,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:45,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:45,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:45,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:45,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:45,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:45,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:45,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +12: [2023-05-13 09:38:45,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +12: [2023-05-13 09:38:45,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:38:45,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:45,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:45,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:45,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +20: [2023-05-13 09:38:45,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +20: [2023-05-13 09:38:45,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +20: [2023-05-13 09:38:45,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:38:45,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:45,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:45,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:45,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:45,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:45,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:45,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +29: [2023-05-13 09:38:45,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:45,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:45,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +21: [2023-05-13 09:38:45,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +21: [2023-05-13 09:38:45,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:45,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:45,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:45,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:45,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:45,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:38:45,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:38:45,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:38:45,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:38:45,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:45,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:45,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:45,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:45,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:38:45,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:45,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:45,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:45,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:45,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:45,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:38:45,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:45,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +10: [2023-05-13 09:38:45,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:45,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:45,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:45,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:45,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:45,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:45,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:45,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:45,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:45,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:45,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:45,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:45,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:45,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:45,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:45,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:45,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:45,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:45,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:45,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:45,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:45,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:45,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 0: > overriding learning rate value to 0.0002 + 0: > overriding minimum learning rate value to 2e-05 + 0: > overriding warmup iterations value to 0 + 0: > overriding total number of iterations value to 1 + 0: > overriding decay style value to cosine +25: [2023-05-13 09:38:45,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:45,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:45,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:45,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:45,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:45,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:45,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:45,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:45,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:45,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:45,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:45,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +29: [2023-05-13 09:38:45,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:45,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:45,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:45,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:45,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:45,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:45,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:45,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:45,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:38:45,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:38:45,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:38:45,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:38:45,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:45,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +10: [2023-05-13 09:38:45,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:45,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:45,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:38:45,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:38:45,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +29: [2023-05-13 09:38:45,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:38:45,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:38:45,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +28: [2023-05-13 09:38:45,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:38:45,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +28: [2023-05-13 09:38:45,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt... +19: [2023-05-13 09:38:45,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt... +19: [2023-05-13 09:38:45,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt... +19: [2023-05-13 09:38:45,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:45,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:45,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:38:45,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:45,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:45,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:38:45,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:38:45,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:38:45,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:45,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:38:45,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:38:45,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:38:45,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:45,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt... +11: [2023-05-13 09:38:45,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:45,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:45,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:38:45,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +14: [2023-05-13 09:38:45,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +14: [2023-05-13 09:38:45,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +14: [2023-05-13 09:38:45,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:38:45,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:45,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:45,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:45,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:45,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +25: [2023-05-13 09:38:45,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:45,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:45,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +25: [2023-05-13 09:38:45,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:45,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:45,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:45,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:38:45,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:45,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:45,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:45,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:45,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:45,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:45,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:45,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:45,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:45,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:45,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:45,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:45,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +19: [2023-05-13 09:38:45,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:38:45,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:45,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +20: [2023-05-13 09:38:45,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:45,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:38:45,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:45,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:45,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:45,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... + 0: [2023-05-13 09:38:45,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... + 0: [2023-05-13 09:38:45,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... + 0: [2023-05-13 09:38:45,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... +20: [2023-05-13 09:38:45,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +13: [2023-05-13 09:38:45,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:45,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:45,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:45,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:45,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:45,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:45,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:45,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:45,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:45,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:45,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:45,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:45,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:45,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:45,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:38:45,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:45,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:45,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:45,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:45,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:38:45,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:45,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:45,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:45,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:45,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:45,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:45,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:45,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:45,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:45,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:45,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:45,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:45,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:45,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:45,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:45,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:45,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:45,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:45,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:38:45,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:38:45,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:38:45,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:38:45,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:38:45,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +27: [2023-05-13 09:38:45,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +27: [2023-05-13 09:38:45,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +27: [2023-05-13 09:38:45,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +16: [2023-05-13 09:38:45,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +16: [2023-05-13 09:38:45,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +16: [2023-05-13 09:38:45,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +16: [2023-05-13 09:38:45,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:45,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:38:45,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:38:45,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:38:45,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:45,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:45,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:45,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:45,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:38:45,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:45,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:45,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:38:45,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:45,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:38:45,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:38:45,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:45,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:38:45,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +10: [2023-05-13 09:38:45,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:45,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:45,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:45,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:45,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:45,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +10: [2023-05-13 09:38:45,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:38:45,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:38:45,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:38:45,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:38:45,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +11: [2023-05-13 09:38:45,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +11: [2023-05-13 09:38:45,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +11: [2023-05-13 09:38:45,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:38:45,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:45,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:45,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:45,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:45,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:45,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:45,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:45,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:45,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:38:45,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt... +28: [2023-05-13 09:38:45,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt... +28: [2023-05-13 09:38:45,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt... +28: [2023-05-13 09:38:45,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:45,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:45,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:45,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +17: [2023-05-13 09:38:45,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +17: [2023-05-13 09:38:45,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +17: [2023-05-13 09:38:45,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +27: [2023-05-13 09:38:45,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +27: [2023-05-13 09:38:45,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +26: [2023-05-13 09:38:45,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +26: [2023-05-13 09:38:45,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +17: [2023-05-13 09:38:45,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:45,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +26: [2023-05-13 09:38:45,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:38:45,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:38:45,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:38:45,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:38:45,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +17: [2023-05-13 09:38:45,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:45,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:45,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:45,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:45,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:45,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:45,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:45,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +27: [2023-05-13 09:38:45,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:45,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +16: [2023-05-13 09:38:45,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +16: [2023-05-13 09:38:45,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:45,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:45,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:45,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:45,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:45,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:45,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:38:45,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:45,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:45,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:45,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:45,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:45,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:45,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:45,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:45,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:45,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:45,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:45,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:45,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +17: [2023-05-13 09:38:45,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:45,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:45,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:45,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:45,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:45,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:38:45,296] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 102 +15: [2023-05-13 09:38:45,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:45,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:45,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:38:45,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:38:45,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:45,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:45,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:45,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:45,301] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 102 + 2: [2023-05-13 09:38:45,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:45,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:38:45,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:38:45,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:45,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:38:45,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:38:45,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:38:45,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +26: [2023-05-13 09:38:45,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +23: [2023-05-13 09:38:45,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:45,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:45,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:45,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:45,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:38:45,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:38:45,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +26: [2023-05-13 09:38:45,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:38:45,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:38:45,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:38:45,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +24: [2023-05-13 09:38:45,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:38:45,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:38:45,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:38:45,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:45,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:45,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:45,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:45,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:38:45,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +14: [2023-05-13 09:38:45,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +14: [2023-05-13 09:38:45,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +14: [2023-05-13 09:38:45,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +14: [2023-05-13 09:38:45,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:38:45,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:45,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:45,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:45,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:45,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:45,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:45,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +31: [2023-05-13 09:38:45,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:45,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +31: [2023-05-13 09:38:45,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:45,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +18: [2023-05-13 09:38:45,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +18: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:45,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:45,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:38:45,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:45,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:45,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:45,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:45,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:38:45,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:45,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:38:45,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:38:45,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:45,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:45,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:45,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:45,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. + 5: [2023-05-13 09:38:45,378] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 46 + 7: [2023-05-13 09:38:45,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:45,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 5: [2023-05-13 09:38:45,385] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 46 +24: [2023-05-13 09:38:45,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +24: [2023-05-13 09:38:45,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:45,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:38:45,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:38:45,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:38:45,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:38:45,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:45,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +24: [2023-05-13 09:38:45,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:45,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:45,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:45,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:45,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:45,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 3: [2023-05-13 09:38:45,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 3: [2023-05-13 09:38:45,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:45,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:45,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:45,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:45,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:45,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 5: [2023-05-13 09:38:45,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:45,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. + 5: [2023-05-13 09:38:45,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:45,407] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 96 + 5: [2023-05-13 09:38:45,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +14: [2023-05-13 09:38:45,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +14: [2023-05-13 09:38:45,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +14: [2023-05-13 09:38:45,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:38:45,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:45,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:38:45,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:45,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +18: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:38:45,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:38:45,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +29: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:38:45,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +12: [2023-05-13 09:38:45,414] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 96 +18: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +29: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:45,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 5: [2023-05-13 09:38:45,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +20: [2023-05-13 09:38:45,418] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 160 +12: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:38:45,419] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 98 + 3: [2023-05-13 09:38:45,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:45,424] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 160 + 3: [2023-05-13 09:38:45,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +12: [2023-05-13 09:38:45,425] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 98 +30: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:45,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:45,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:45,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:45,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:45,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:45,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:45,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:45,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +22: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +29: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:45,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:45,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:45,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:45,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:38:45,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +29: [2023-05-13 09:38:45,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +29: [2023-05-13 09:38:45,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 3: [2023-05-13 09:38:45,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 3: [2023-05-13 09:38:45,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:38:45,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:38:45,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:38:45,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:38:45,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:45,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:38:45,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:38:45,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:38:45,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:45,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:38:45,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +13: [2023-05-13 09:38:45,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:38:45,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +29: [2023-05-13 09:38:45,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +29: [2023-05-13 09:38:45,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. + 5: [2023-05-13 09:38:45,445] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 44 +13: [2023-05-13 09:38:45,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:38:45,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,452] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 44 + 6: [2023-05-13 09:38:45,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:45,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:45,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:45,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:38:45,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:45,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:38:45,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:45,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:45,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:45,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:38:45,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:38:45,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:38:45,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +20: [2023-05-13 09:38:45,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt. +20: [2023-05-13 09:38:45,463] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 166 +27: [2023-05-13 09:38:45,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:38:45,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:38:45,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:38:45,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:38:45,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:45,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:45,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +30: [2023-05-13 09:38:45,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:38:45,470] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 166 + 5: [2023-05-13 09:38:45,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. + 5: [2023-05-13 09:38:45,471] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 42 +24: [2023-05-13 09:38:45,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt. + 6: [2023-05-13 09:38:45,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:45,474] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 198 +31: [2023-05-13 09:38:45,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:38:45,476] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 252 +10: [2023-05-13 09:38:45,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:45,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +19: [2023-05-13 09:38:45,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +19: [2023-05-13 09:38:45,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +19: [2023-05-13 09:38:45,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:38:45,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:38:45,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:38:45,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:38:45,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:38:45,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. + 4: [2023-05-13 09:38:45,479] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 38 + 5: [2023-05-13 09:38:45,479] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 42 + 6: [2023-05-13 09:38:45,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:45,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:45,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:45,480] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 198 +10: [2023-05-13 09:38:45,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:45,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:45,483] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 252 +10: [2023-05-13 09:38:45,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:45,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,485] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 38 +25: [2023-05-13 09:38:45,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:45,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +19: [2023-05-13 09:38:45,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:45,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:45,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:45,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:45,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:45,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:45,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:45,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:45,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:45,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:38:45,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +29: [2023-05-13 09:38:45,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt... +29: [2023-05-13 09:38:45,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt... +29: [2023-05-13 09:38:45,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt... +29: [2023-05-13 09:38:45,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:38:45,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:38:45,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:45,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 3: [2023-05-13 09:38:45,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. + 3: [2023-05-13 09:38:45,508] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 28 + 2: [2023-05-13 09:38:45,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:38:45,509] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 18 +27: [2023-05-13 09:38:45,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:45,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +17: [2023-05-13 09:38:45,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:38:45,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:45,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:45,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:38:45,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:45,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:38:45,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,515] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 28 + 6: [2023-05-13 09:38:45,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +27: [2023-05-13 09:38:45,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,518] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 18 + 6: [2023-05-13 09:38:45,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:45,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +19: [2023-05-13 09:38:45,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:45,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:38:45,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:45,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:38:45,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:38:45,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:38:45,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:38:45,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:45,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:45,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:45,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:38:45,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +19: [2023-05-13 09:38:45,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:45,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:38:45,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:38:45,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:38:45,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:38:45,525] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 236 +17: [2023-05-13 09:38:45,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:45,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:38:45,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:45,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:45,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:38:45,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:45,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:38:45,532] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 236 +11: [2023-05-13 09:38:45,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:38:45,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:38:45,537] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 144 +19: [2023-05-13 09:38:45,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +13: [2023-05-13 09:38:45,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +13: [2023-05-13 09:38:45,544] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 108 +18: [2023-05-13 09:38:45,546] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 144 +31: [2023-05-13 09:38:45,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:38:45,549] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 248 +13: [2023-05-13 09:38:45,551] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 108 + 3: [2023-05-13 09:38:45,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. + 3: [2023-05-13 09:38:45,553] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 24 +23: [2023-05-13 09:38:45,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:38:45,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:38:45,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +29: [2023-05-13 09:38:45,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:38:45,554] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 234 +23: [2023-05-13 09:38:45,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:38:45,555] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 248 +17: [2023-05-13 09:38:45,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:45,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt. +23: [2023-05-13 09:38:45,560] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 190 + 3: [2023-05-13 09:38:45,560] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 24 +29: [2023-05-13 09:38:45,562] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 234 +17: [2023-05-13 09:38:45,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:45,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:38:45,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:45,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. + 1: [2023-05-13 09:38:45,565] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 8 + 2: [2023-05-13 09:38:45,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:38:45,565] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 16 +17: [2023-05-13 09:38:45,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:38:45,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:45,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:45,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:45,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:45,569] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 190 +17: [2023-05-13 09:38:45,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,571] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 8 + 2: [2023-05-13 09:38:45,572] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 16 +28: [2023-05-13 09:38:45,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:38:45,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:38:45,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:38:45,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:38:45,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +20: [2023-05-13 09:38:45,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:38:45,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +20: [2023-05-13 09:38:45,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +20: [2023-05-13 09:38:45,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +20: [2023-05-13 09:38:45,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:38:45,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:38:45,577] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 134 +18: [2023-05-13 09:38:45,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:38:45,577] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 150 +21: [2023-05-13 09:38:45,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:38:45,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:38:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt. +24: [2023-05-13 09:38:45,583] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 194 +16: [2023-05-13 09:38:45,584] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 134 +18: [2023-05-13 09:38:45,584] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 150 +23: [2023-05-13 09:38:45,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt. +23: [2023-05-13 09:38:45,585] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 186 +26: [2023-05-13 09:38:45,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:38:45,586] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 210 +18: [2023-05-13 09:38:45,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:38:45,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:38:45,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:45,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +16: [2023-05-13 09:38:45,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +18: [2023-05-13 09:38:45,587] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 146 +28: [2023-05-13 09:38:45,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:38:45,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 3: [2023-05-13 09:38:45,588] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 30 +28: [2023-05-13 09:38:45,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +24: [2023-05-13 09:38:45,589] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 194 +28: [2023-05-13 09:38:45,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:45,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:45,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:45,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:45,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +20: [2023-05-13 09:38:45,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:45,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:45,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:45,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:45,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:45,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:45,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:38:45,591] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 186 +20: [2023-05-13 09:38:45,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +20: [2023-05-13 09:38:45,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +28: [2023-05-13 09:38:45,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +28: [2023-05-13 09:38:45,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +20: [2023-05-13 09:38:45,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:45,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +20: [2023-05-13 09:38:45,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:45,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:45,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +28: [2023-05-13 09:38:45,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +12: [2023-05-13 09:38:45,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +12: [2023-05-13 09:38:45,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +12: [2023-05-13 09:38:45,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +12: [2023-05-13 09:38:45,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:38:45,594] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 146 +28: [2023-05-13 09:38:45,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:45,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +20: [2023-05-13 09:38:45,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +20: [2023-05-13 09:38:45,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +28: [2023-05-13 09:38:45,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:45,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +20: [2023-05-13 09:38:45,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:45,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +15: [2023-05-13 09:38:45,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +26: [2023-05-13 09:38:45,595] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 210 +21: [2023-05-13 09:38:45,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:45,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 3: [2023-05-13 09:38:45,595] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 30 +21: [2023-05-13 09:38:45,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:45,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:45,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:38:45,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt. +24: [2023-05-13 09:38:45,596] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 192 + 2: [2023-05-13 09:38:45,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:38:45,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +16: [2023-05-13 09:38:45,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. +23: [2023-05-13 09:38:45,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,601] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 40 +24: [2023-05-13 09:38:45,603] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 192 +23: [2023-05-13 09:38:45,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:38:45,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:45,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:45,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:45,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:45,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:45,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt. +24: [2023-05-13 09:38:45,607] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 196 +12: [2023-05-13 09:38:45,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,607] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 40 +12: [2023-05-13 09:38:45,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:45,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:45,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +12: [2023-05-13 09:38:45,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:38:45,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:45,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:45,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:45,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:38:45,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:45,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:38:45,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:45,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:45,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:45,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:45,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +12: [2023-05-13 09:38:45,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:45,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:45,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:45,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:38:45,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:45,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:45,614] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 196 + 2: [2023-05-13 09:38:45,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:45,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:38:45,621] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 22 + 4: [2023-05-13 09:38:45,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:38:45,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:45,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:38:45,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +23: [2023-05-13 09:38:45,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +31: [2023-05-13 09:38:45,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:38:45,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:38:45,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:38:45,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:45,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,628] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 22 +14: [2023-05-13 09:38:45,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +14: [2023-05-13 09:38:45,628] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 114 +16: [2023-05-13 09:38:45,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:45,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:38:45,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:38:45,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:45,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +20: [2023-05-13 09:38:45,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt... +20: [2023-05-13 09:38:45,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt... +20: [2023-05-13 09:38:45,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt... +20: [2023-05-13 09:38:45,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt... +14: [2023-05-13 09:38:45,635] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 114 + 1: [2023-05-13 09:38:45,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:38:45,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:45,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:38:45,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +31: [2023-05-13 09:38:45,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:45,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:45,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:45,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +31: [2023-05-13 09:38:45,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:45,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:45,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:45,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:45,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt... +12: [2023-05-13 09:38:45,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:38:45,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:45,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt... +12: [2023-05-13 09:38:45,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt... +15: [2023-05-13 09:38:45,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:45,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +13: [2023-05-13 09:38:45,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +13: [2023-05-13 09:38:45,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +15: [2023-05-13 09:38:45,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +13: [2023-05-13 09:38:45,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +13: [2023-05-13 09:38:45,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +26: [2023-05-13 09:38:45,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +26: [2023-05-13 09:38:45,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +26: [2023-05-13 09:38:45,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:45,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:38:45,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:45,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +15: [2023-05-13 09:38:45,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:38:45,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +24: [2023-05-13 09:38:45,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:38:45,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:38:45,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:38:45,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +18: [2023-05-13 09:38:45,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +18: [2023-05-13 09:38:45,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +18: [2023-05-13 09:38:45,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:38:45,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:38:45,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:38:45,665] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 230 +26: [2023-05-13 09:38:45,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:45,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt... +28: [2023-05-13 09:38:45,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt... +28: [2023-05-13 09:38:45,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt... +28: [2023-05-13 09:38:45,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:38:45,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:45,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:45,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:45,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:45,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:45,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:45,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:45,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:45,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:45,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:45,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:45,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:45,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:45,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:45,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:45,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:38:45,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:38:45,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +26: [2023-05-13 09:38:45,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +26: [2023-05-13 09:38:45,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +26: [2023-05-13 09:38:45,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +26: [2023-05-13 09:38:45,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +13: [2023-05-13 09:38:45,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:45,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:45,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt. +22: [2023-05-13 09:38:45,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +22: [2023-05-13 09:38:45,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:38:45,672] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 188 +22: [2023-05-13 09:38:45,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +22: [2023-05-13 09:38:45,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:45,673] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 230 +14: [2023-05-13 09:38:45,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +14: [2023-05-13 09:38:45,673] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 116 +13: [2023-05-13 09:38:45,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:38:45,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:38:45,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +26: [2023-05-13 09:38:45,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +26: [2023-05-13 09:38:45,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +26: [2023-05-13 09:38:45,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:38:45,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:38:45,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:38:45,674] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 184 +31: [2023-05-13 09:38:45,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +13: [2023-05-13 09:38:45,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +24: [2023-05-13 09:38:45,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:38:45,676] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 220 +31: [2023-05-13 09:38:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. + 1: [2023-05-13 09:38:45,677] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 10 + 8: [2023-05-13 09:38:45,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:45,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:45,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:45,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. + 6: [2023-05-13 09:38:45,679] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 52 +18: [2023-05-13 09:38:45,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:45,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:45,680] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 188 +31: [2023-05-13 09:38:45,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:45,681] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 184 +18: [2023-05-13 09:38:45,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:45,681] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 116 +18: [2023-05-13 09:38:45,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:45,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:45,682] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 220 +31: [2023-05-13 09:38:45,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +14: [2023-05-13 09:38:45,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +14: [2023-05-13 09:38:45,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +14: [2023-05-13 09:38:45,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:38:45,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:45,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,684] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 10 +14: [2023-05-13 09:38:45,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +29: [2023-05-13 09:38:45,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:38:45,685] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 238 + 6: [2023-05-13 09:38:45,686] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 52 +22: [2023-05-13 09:38:45,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:45,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:45,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:45,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:45,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +13: [2023-05-13 09:38:45,690] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 110 + 2: [2023-05-13 09:38:45,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:38:45,691] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 20 +29: [2023-05-13 09:38:45,695] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 238 +14: [2023-05-13 09:38:45,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:45,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:45,697] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 110 + 2: [2023-05-13 09:38:45,698] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 20 + 9: [2023-05-13 09:38:45,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:45,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:45,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:45,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +14: [2023-05-13 09:38:45,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +14: [2023-05-13 09:38:45,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:45,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:45,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +14: [2023-05-13 09:38:45,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:45,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:38:45,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:45,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +14: [2023-05-13 09:38:45,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +14: [2023-05-13 09:38:45,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +14: [2023-05-13 09:38:45,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:38:45,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt. +22: [2023-05-13 09:38:45,705] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 176 +18: [2023-05-13 09:38:45,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +24: [2023-05-13 09:38:45,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +25: [2023-05-13 09:38:45,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +25: [2023-05-13 09:38:45,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:45,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt. + 4: [2023-05-13 09:38:45,707] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 34 +19: [2023-05-13 09:38:45,708] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 156 + 9: [2023-05-13 09:38:45,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:38:45,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:38:45,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:38:45,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:38:45,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:38:45,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:38:45,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt. + 0: [2023-05-13 09:38:45,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:38:45,710] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 128 + 0: [2023-05-13 09:38:45,711] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 0 + 9: [2023-05-13 09:38:45,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:38:45,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:38:45,711] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 176 + 8: [2023-05-13 09:38:45,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:45,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:45,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:45,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,714] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 156 +18: [2023-05-13 09:38:45,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:45,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +10: [2023-05-13 09:38:45,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +10: [2023-05-13 09:38:45,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:38:45,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:45,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:38:45,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,715] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 34 +24: [2023-05-13 09:38:45,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +13: [2023-05-13 09:38:45,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:38:45,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:38:45,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +13: [2023-05-13 09:38:45,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:38:45,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:38:45,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:38:45,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:38:45,716] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 208 +16: [2023-05-13 09:38:45,717] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 128 +18: [2023-05-13 09:38:45,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,719] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 0 +30: [2023-05-13 09:38:45,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:38:45,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:38:45,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:38:45,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +22: [2023-05-13 09:38:45,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt... + 4: [2023-05-13 09:38:45,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt... + 4: [2023-05-13 09:38:45,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:38:45,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:38:45,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:38:45,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:38:45,722] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 232 + 0: [2023-05-13 09:38:45,722] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 6 +26: [2023-05-13 09:38:45,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:38:45,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:38:45,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:38:45,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:38:45,723] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 208 + 5: [2023-05-13 09:38:45,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:45,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:45,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:45,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:45,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:45,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:45,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:38:45,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:45,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +25: [2023-05-13 09:38:45,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +25: [2023-05-13 09:38:45,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 0: could not find arguments in the checkpoint ... + 0: checkpoint version 3.0 + 4: [2023-05-13 09:38:45,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. + 4: [2023-05-13 09:38:45,726] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 32 +18: [2023-05-13 09:38:45,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:38:45,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +25: [2023-05-13 09:38:45,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +25: [2023-05-13 09:38:45,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:38:45,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:45,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:45,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:45,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:45,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:45,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +25: [2023-05-13 09:38:45,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,729] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 6 +29: [2023-05-13 09:38:45,730] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 232 +22: [2023-05-13 09:38:45,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +22: [2023-05-13 09:38:45,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +22: [2023-05-13 09:38:45,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,733] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 32 +21: [2023-05-13 09:38:45,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt. +21: [2023-05-13 09:38:45,734] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 168 +10: [2023-05-13 09:38:45,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:45,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:45,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:45,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:45,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:45,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:45,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:45,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:45,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:45,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:45,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +10: [2023-05-13 09:38:45,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +10: [2023-05-13 09:38:45,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +10: [2023-05-13 09:38:45,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +27: [2023-05-13 09:38:45,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +27: [2023-05-13 09:38:45,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +27: [2023-05-13 09:38:45,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +27: [2023-05-13 09:38:45,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:38:45,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:45,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:45,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +10: [2023-05-13 09:38:45,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +10: [2023-05-13 09:38:45,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +10: [2023-05-13 09:38:45,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,741] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 168 +14: [2023-05-13 09:38:45,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt... +14: [2023-05-13 09:38:45,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt... +14: [2023-05-13 09:38:45,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt... +14: [2023-05-13 09:38:45,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:38:45,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:38:45,744] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 250 +21: [2023-05-13 09:38:45,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt. +21: [2023-05-13 09:38:45,745] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 174 + 6: [2023-05-13 09:38:45,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:45,750] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 250 + 6: [2023-05-13 09:38:45,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:45,752] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 174 +27: [2023-05-13 09:38:45,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:45,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:45,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:38:45,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:38:45,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:45,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:38:45,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:45,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:45,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +27: [2023-05-13 09:38:45,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +27: [2023-05-13 09:38:45,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +27: [2023-05-13 09:38:45,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:38:45,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt. +19: [2023-05-13 09:38:45,758] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 158 +27: [2023-05-13 09:38:45,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:38:45,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:45,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:38:45,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 5: [2023-05-13 09:38:45,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,765] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 158 +30: [2023-05-13 09:38:45,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:38:45,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:38:45,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:45,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:45,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +30: [2023-05-13 09:38:45,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:38:45,769] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 92 +30: [2023-05-13 09:38:45,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 5: [2023-05-13 09:38:45,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:38:45,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:38:45,771] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 228 + 4: [2023-05-13 09:38:45,771] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 36 + 6: [2023-05-13 09:38:45,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:38:45,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:45,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt... +25: [2023-05-13 09:38:45,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt... +25: [2023-05-13 09:38:45,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:38:45,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:45,775] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 92 + 6: [2023-05-13 09:38:45,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:45,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:38:45,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:38:45,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:38:45,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,778] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 36 +11: [2023-05-13 09:38:45,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:38:45,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt. + 6: [2023-05-13 09:38:45,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:38:45,778] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 212 +11: [2023-05-13 09:38:45,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:38:45,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:38:45,779] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 228 +26: [2023-05-13 09:38:45,785] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 212 +30: [2023-05-13 09:38:45,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt. +30: [2023-05-13 09:38:45,787] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 246 + 9: [2023-05-13 09:38:45,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. + 9: [2023-05-13 09:38:45,791] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 78 +10: [2023-05-13 09:38:45,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt... +10: [2023-05-13 09:38:45,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt... +10: [2023-05-13 09:38:45,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt... +10: [2023-05-13 09:38:45,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:38:45,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:38:45,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt. + 3: [2023-05-13 09:38:45,793] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 26 +18: [2023-05-13 09:38:45,793] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 148 +11: [2023-05-13 09:38:45,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:45,794] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 246 +11: [2023-05-13 09:38:45,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:45,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:45,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:45,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +11: [2023-05-13 09:38:45,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +11: [2023-05-13 09:38:45,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +11: [2023-05-13 09:38:45,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,798] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 78 + 9: [2023-05-13 09:38:45,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. +21: [2023-05-13 09:38:45,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt. +21: [2023-05-13 09:38:45,799] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 170 + 9: [2023-05-13 09:38:45,799] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 72 +16: [2023-05-13 09:38:45,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:38:45,800] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 132 +18: [2023-05-13 09:38:45,800] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 148 + 7: [2023-05-13 09:38:45,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,802] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 26 + 7: [2023-05-13 09:38:45,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,805] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 72 +21: [2023-05-13 09:38:45,805] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 170 +11: [2023-05-13 09:38:45,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +11: [2023-05-13 09:38:45,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +11: [2023-05-13 09:38:45,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +11: [2023-05-13 09:38:45,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:38:45,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:38:45,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:38:45,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:38:45,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:38:45,806] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 132 +11: [2023-05-13 09:38:45,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +11: [2023-05-13 09:38:45,808] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 88 + 7: [2023-05-13 09:38:45,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:45,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +13: [2023-05-13 09:38:45,810] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 106 +22: [2023-05-13 09:38:45,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt. +22: [2023-05-13 09:38:45,811] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 178 + 7: [2023-05-13 09:38:45,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. + 7: [2023-05-13 09:38:45,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,811] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 60 + 7: [2023-05-13 09:38:45,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:45,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:45,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:45,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:45,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:38:45,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +11: [2023-05-13 09:38:45,814] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 94 + 7: [2023-05-13 09:38:45,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt... +20: [2023-05-13 09:38:45,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt. +20: [2023-05-13 09:38:45,816] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 162 + 9: [2023-05-13 09:38:45,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. + 9: [2023-05-13 09:38:45,816] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 76 +11: [2023-05-13 09:38:45,817] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 88 +22: [2023-05-13 09:38:45,818] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 178 +13: [2023-05-13 09:38:45,818] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 106 + 7: [2023-05-13 09:38:45,818] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 60 +11: [2023-05-13 09:38:45,821] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 94 +20: [2023-05-13 09:38:45,822] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 162 + 9: [2023-05-13 09:38:45,823] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 76 +16: [2023-05-13 09:38:45,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:38:45,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:45,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:45,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:38:45,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:45,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:45,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:45,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt. +22: [2023-05-13 09:38:45,834] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 180 + 2: [2023-05-13 09:38:45,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:45,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:45,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt. +21: [2023-05-13 09:38:45,839] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 172 +23: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:38:45,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. +23: [2023-05-13 09:38:45,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,840] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 50 +22: [2023-05-13 09:38:45,841] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 180 + 2: [2023-05-13 09:38:45,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:45,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:38:45,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:38:45,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:38:45,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:45,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,844] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 58 +16: [2023-05-13 09:38:45,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,845] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 172 +16: [2023-05-13 09:38:45,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:45,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:45,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:38:45,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:38:45,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:45,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:38:45,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:45,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:38:45,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt. +30: [2023-05-13 09:38:45,848] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 240 +16: [2023-05-13 09:38:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:38:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:38:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:38:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:45,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt. +19: [2023-05-13 09:38:45,849] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 154 + 6: [2023-05-13 09:38:45,849] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 50 + 2: [2023-05-13 09:38:45,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,852] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 58 +11: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt... +11: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt... +11: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt... +11: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:38:45,855] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 240 +19: [2023-05-13 09:38:45,856] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 154 +23: [2023-05-13 09:38:45,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:45,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:38:45,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:45,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:38:45,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:45,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:45,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:45,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:38:45,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:45,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:45,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:38:45,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:38:45,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:38:45,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. +14: [2023-05-13 09:38:45,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. + 1: [2023-05-13 09:38:45,862] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 14 +14: [2023-05-13 09:38:45,862] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 118 +15: [2023-05-13 09:38:45,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +15: [2023-05-13 09:38:45,863] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 122 + 1: [2023-05-13 09:38:45,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. + 1: [2023-05-13 09:38:45,864] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 12 +15: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +15: [2023-05-13 09:38:45,864] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 126 + 1: [2023-05-13 09:38:45,869] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 14 +15: [2023-05-13 09:38:45,869] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 122 + 1: [2023-05-13 09:38:45,871] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 12 +15: [2023-05-13 09:38:45,871] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 126 +14: [2023-05-13 09:38:45,871] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 118 +17: [2023-05-13 09:38:45,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt. +17: [2023-05-13 09:38:45,873] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 140 + 1: [2023-05-13 09:38:45,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:45,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt. +19: [2023-05-13 09:38:45,876] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 152 + 1: [2023-05-13 09:38:45,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:38:45,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:38:45,877] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 130 +17: [2023-05-13 09:38:45,879] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 140 + 7: [2023-05-13 09:38:45,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. + 7: [2023-05-13 09:38:45,880] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 56 +17: [2023-05-13 09:38:45,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:38:45,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:38:45,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:38:45,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:45,884] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 152 +16: [2023-05-13 09:38:45,885] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 130 + 7: [2023-05-13 09:38:45,887] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 56 +16: [2023-05-13 09:38:45,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:38:45,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:38:45,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:38:45,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:38:45,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt... +19: [2023-05-13 09:38:45,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt... +19: [2023-05-13 09:38:45,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:38:45,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:45,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:38:45,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt... + 2: [2023-05-13 09:38:45,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt... + 2: [2023-05-13 09:38:45,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt... + 2: [2023-05-13 09:38:45,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:38:45,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:38:45,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:38:45,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:38:45,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:38:45,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:38:45,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:45,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:45,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +17: [2023-05-13 09:38:45,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:45,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:45,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +17: [2023-05-13 09:38:45,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:45,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:45,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +17: [2023-05-13 09:38:45,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:45,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:45,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +17: [2023-05-13 09:38:45,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +17: [2023-05-13 09:38:45,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +17: [2023-05-13 09:38:45,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +17: [2023-05-13 09:38:45,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +17: [2023-05-13 09:38:45,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:38:45,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:38:45,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:38:45,900] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 142 +18: [2023-05-13 09:38:45,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt. +30: [2023-05-13 09:38:45,907] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 242 +17: [2023-05-13 09:38:45,907] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 142 +24: [2023-05-13 09:38:45,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:38:45,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. + 7: [2023-05-13 09:38:45,911] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 62 +24: [2023-05-13 09:38:45,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:38:45,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:38:45,914] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 242 +20: [2023-05-13 09:38:45,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt. +20: [2023-05-13 09:38:45,915] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 164 +18: [2023-05-13 09:38:45,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:45,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:45,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:45,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:45,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:38:45,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:45,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:38:45,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:38:45,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,917] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 182 +18: [2023-05-13 09:38:45,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:38:45,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +15: [2023-05-13 09:38:45,918] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 124 +18: [2023-05-13 09:38:45,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:45,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +18: [2023-05-13 09:38:45,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +18: [2023-05-13 09:38:45,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:45,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:38:45,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,919] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 62 +18: [2023-05-13 09:38:45,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:45,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +15: [2023-05-13 09:38:45,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +15: [2023-05-13 09:38:45,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:38:45,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +11: [2023-05-13 09:38:45,922] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 90 +20: [2023-05-13 09:38:45,922] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 164 +15: [2023-05-13 09:38:45,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:38:45,924] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 182 +15: [2023-05-13 09:38:45,924] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 124 +24: [2023-05-13 09:38:45,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:38:45,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:38:45,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:38:45,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:38:45,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:45,929] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 90 +15: [2023-05-13 09:38:45,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +24: [2023-05-13 09:38:45,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:38:45,929] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 120 +21: [2023-05-13 09:38:45,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:38:45,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:38:45,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:38:45,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:38:45,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:38:45,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:45,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:45,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +24: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +31: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:38:45,932] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 254 +24: [2023-05-13 09:38:45,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +24: [2023-05-13 09:38:45,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:45,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:45,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +24: [2023-05-13 09:38:45,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:45,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:45,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:45,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. +15: [2023-05-13 09:38:45,937] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 120 +15: [2023-05-13 09:38:45,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:45,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,938] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 74 +15: [2023-05-13 09:38:45,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:45,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:38:45,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:38:45,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:38:45,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:45,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:45,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +31: [2023-05-13 09:38:45,939] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 254 +15: [2023-05-13 09:38:45,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:45,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:45,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:45,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:38:45,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,945] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 74 +22: [2023-05-13 09:38:45,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:45,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:45,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:45,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:45,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:45,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:45,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:45,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:45,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:38:45,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:45,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:45,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:38:45,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +21: [2023-05-13 09:38:45,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +21: [2023-05-13 09:38:45,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +21: [2023-05-13 09:38:45,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:38:45,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:45,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:45,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:45,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:38:45,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:38:45,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:38:45,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +21: [2023-05-13 09:38:45,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +17: [2023-05-13 09:38:45,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:38:45,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:38:45,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:38:45,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:38:45,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt. +17: [2023-05-13 09:38:45,955] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 136 +18: [2023-05-13 09:38:45,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt... +18: [2023-05-13 09:38:45,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt... +18: [2023-05-13 09:38:45,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt... +18: [2023-05-13 09:38:45,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt... +28: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:38:45,959] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 226 + 8: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. + 8: [2023-05-13 09:38:45,959] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 70 +17: [2023-05-13 09:38:45,961] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 136 +26: [2023-05-13 09:38:45,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:38:45,962] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 214 +28: [2023-05-13 09:38:45,965] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 226 + 8: [2023-05-13 09:38:45,966] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 70 + 8: [2023-05-13 09:38:45,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +26: [2023-05-13 09:38:45,969] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 214 + 8: [2023-05-13 09:38:45,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:38:45,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:38:45,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:38:45,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:38:45,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:38:45,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:38:45,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:38:45,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:38:45,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +15: [2023-05-13 09:38:45,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt... +15: [2023-05-13 09:38:45,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt... +15: [2023-05-13 09:38:45,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt... +15: [2023-05-13 09:38:45,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt... +28: [2023-05-13 09:38:45,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:38:45,979] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 224 +22: [2023-05-13 09:38:45,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt... +22: [2023-05-13 09:38:45,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt... +22: [2023-05-13 09:38:45,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt... +22: [2023-05-13 09:38:45,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt... + 8: [2023-05-13 09:38:45,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,986] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 68 + 8: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:45,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:45,986] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 224 +31: [2023-05-13 09:38:45,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +31: [2023-05-13 09:38:45,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +31: [2023-05-13 09:38:45,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +31: [2023-05-13 09:38:45,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:45,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:45,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt... +21: [2023-05-13 09:38:45,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:38:45,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:45,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt... +21: [2023-05-13 09:38:45,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:38:45,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +31: [2023-05-13 09:38:45,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +31: [2023-05-13 09:38:45,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +31: [2023-05-13 09:38:45,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,993] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 68 + 8: [2023-05-13 09:38:45,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. + 8: [2023-05-13 09:38:45,994] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 66 + 8: [2023-05-13 09:38:45,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. + 8: [2023-05-13 09:38:45,994] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 64 + 8: [2023-05-13 09:38:46,000] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 66 + 8: [2023-05-13 09:38:46,001] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 64 + 7: [2023-05-13 09:38:46,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:46,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:46,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:38:46,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:38:46,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:46,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:38:46,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:46,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:38:46,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +31: [2023-05-13 09:38:46,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:38:46,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:38:46,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:38:46,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt... + 8: [2023-05-13 09:38:46,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt... + 8: [2023-05-13 09:38:46,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt... + 8: [2023-05-13 09:38:46,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt... + 8: [2023-05-13 09:38:46,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:38:46,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:38:46,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 9: [2023-05-13 09:38:46,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 9: [2023-05-13 09:38:46,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 9: [2023-05-13 09:38:46,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 9: [2023-05-13 09:38:46,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:38:46,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:38:46,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +29: [2023-05-13 09:38:46,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt. +29: [2023-05-13 09:38:46,032] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 239 +29: [2023-05-13 09:38:46,038] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 239 +17: [2023-05-13 09:38:46,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt. +17: [2023-05-13 09:38:46,040] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 138 + 6: [2023-05-13 09:38:46,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 6: [2023-05-13 09:38:46,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 6: [2023-05-13 09:38:46,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 6: [2023-05-13 09:38:46,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +27: [2023-05-13 09:38:46,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:38:46,046] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 222 + 9: [2023-05-13 09:38:46,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:46,047] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 138 + 9: [2023-05-13 09:38:46,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:46,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +13: [2023-05-13 09:38:46,047] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 104 + 5: [2023-05-13 09:38:46,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:46,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:38:46,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:38:46,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:46,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:46,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:38:46,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 5: [2023-05-13 09:38:46,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 5: [2023-05-13 09:38:46,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 5: [2023-05-13 09:38:46,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:38:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 9: [2023-05-13 09:38:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 9: [2023-05-13 09:38:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:46,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:38:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:38:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:38:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:38:46,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:38:46,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:46,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:46,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:46,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:38:46,053] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 222 + 7: [2023-05-13 09:38:46,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt... + 7: [2023-05-13 09:38:46,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt... + 7: [2023-05-13 09:38:46,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt... + 7: [2023-05-13 09:38:46,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:38:46,055] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 104 + 6: [2023-05-13 09:38:46,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 6: [2023-05-13 09:38:46,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 6: [2023-05-13 09:38:46,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 6: [2023-05-13 09:38:46,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... + 6: [2023-05-13 09:38:46,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:38:46,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:38:46,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:38:46,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +10: [2023-05-13 09:38:46,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +10: [2023-05-13 09:38:46,065] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 86 +30: [2023-05-13 09:38:46,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:46,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:46,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:46,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:46,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:46,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:46,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:46,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:38:46,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:38:46,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:38:46,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:46,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt... +10: [2023-05-13 09:38:46,072] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 86 +30: [2023-05-13 09:38:46,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:38:46,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:38:46,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:38:46,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/layer_40-model_01-model_states.pt. +29: [2023-05-13 09:38:46,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt. +29: [2023-05-13 09:38:46,076] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 235 + 3: [2023-05-13 09:38:46,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt. +29: [2023-05-13 09:38:46,083] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 235 + 3: [2023-05-13 09:38:46,083] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 29 + 9: [2023-05-13 09:38:46,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt... + 9: [2023-05-13 09:38:46,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt... + 9: [2023-05-13 09:38:46,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt... + 9: [2023-05-13 09:38:46,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:38:46,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:38:46,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:38:46,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:38:46,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:38:46,090] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 29 +12: [2023-05-13 09:38:46,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt. +12: [2023-05-13 09:38:46,093] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 97 +12: [2023-05-13 09:38:46,099] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 97 +10: [2023-05-13 09:38:46,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +10: [2023-05-13 09:38:46,103] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 82 + 6: [2023-05-13 09:38:46,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:38:46,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:38:46,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:38:46,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:38:46,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. + 6: [2023-05-13 09:38:46,109] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 48 +10: [2023-05-13 09:38:46,109] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 82 +30: [2023-05-13 09:38:46,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt... +30: [2023-05-13 09:38:46,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt... +30: [2023-05-13 09:38:46,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt... +30: [2023-05-13 09:38:46,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:38:46,116] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 48 + 6: [2023-05-13 09:38:46,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. + 6: [2023-05-13 09:38:46,117] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 54 + 6: [2023-05-13 09:38:46,126] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 54 +10: [2023-05-13 09:38:46,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +10: [2023-05-13 09:38:46,134] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 84 +10: [2023-05-13 09:38:46,141] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 84 +29: [2023-05-13 09:38:46,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt. +29: [2023-05-13 09:38:46,142] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 237 +28: [2023-05-13 09:38:46,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt. +28: [2023-05-13 09:38:46,146] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 227 +29: [2023-05-13 09:38:46,148] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 237 +28: [2023-05-13 09:38:46,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt. +28: [2023-05-13 09:38:46,148] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 225 +14: [2023-05-13 09:38:46,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt. +14: [2023-05-13 09:38:46,149] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 117 + 3: [2023-05-13 09:38:46,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt. + 3: [2023-05-13 09:38:46,151] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 25 +28: [2023-05-13 09:38:46,153] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 227 +28: [2023-05-13 09:38:46,155] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 225 +14: [2023-05-13 09:38:46,155] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 117 + 3: [2023-05-13 09:38:46,157] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 25 +28: [2023-05-13 09:38:46,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt. +28: [2023-05-13 09:38:46,158] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 229 +28: [2023-05-13 09:38:46,166] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 229 +29: [2023-05-13 09:38:46,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt. +29: [2023-05-13 09:38:46,167] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 233 +29: [2023-05-13 09:38:46,175] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 233 +10: [2023-05-13 09:38:46,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:38:46,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt. +10: [2023-05-13 09:38:46,184] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 80 +27: [2023-05-13 09:38:46,185] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 216 +25: [2023-05-13 09:38:46,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt. +25: [2023-05-13 09:38:46,186] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 200 +27: [2023-05-13 09:38:46,191] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 216 +10: [2023-05-13 09:38:46,192] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 80 +25: [2023-05-13 09:38:46,193] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 200 +30: [2023-05-13 09:38:46,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt. +30: [2023-05-13 09:38:46,200] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 244 + 3: [2023-05-13 09:38:46,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt. + 3: [2023-05-13 09:38:46,202] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 27 +30: [2023-05-13 09:38:46,207] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 244 + 4: [2023-05-13 09:38:46,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt. + 4: [2023-05-13 09:38:46,208] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 35 + 3: [2023-05-13 09:38:46,208] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 27 + 4: [2023-05-13 09:38:46,214] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 35 +14: [2023-05-13 09:38:46,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt. +14: [2023-05-13 09:38:46,221] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 113 +14: [2023-05-13 09:38:46,227] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 113 +20: [2023-05-13 09:38:46,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt. +20: [2023-05-13 09:38:46,229] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 167 +20: [2023-05-13 09:38:46,235] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 167 + 4: [2023-05-13 09:38:46,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt. + 4: [2023-05-13 09:38:46,236] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 39 +27: [2023-05-13 09:38:46,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:38:46,238] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 218 + 4: [2023-05-13 09:38:46,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt. + 4: [2023-05-13 09:38:46,239] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 37 + 4: [2023-05-13 09:38:46,243] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 39 +27: [2023-05-13 09:38:46,245] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 218 + 4: [2023-05-13 09:38:46,246] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 37 +27: [2023-05-13 09:38:46,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt. +27: [2023-05-13 09:38:46,249] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 217 + 4: [2023-05-13 09:38:46,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt. + 4: [2023-05-13 09:38:46,253] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 33 +27: [2023-05-13 09:38:46,256] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 217 + 4: [2023-05-13 09:38:46,260] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 33 + 2: [2023-05-13 09:38:46,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:38:46,267] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 17 +14: [2023-05-13 09:38:46,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +14: [2023-05-13 09:38:46,270] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 112 +12: [2023-05-13 09:38:46,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt. +12: [2023-05-13 09:38:46,271] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 99 + 2: [2023-05-13 09:38:46,274] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 17 +14: [2023-05-13 09:38:46,276] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 112 +12: [2023-05-13 09:38:46,277] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 99 +26: [2023-05-13 09:38:46,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt. +26: [2023-05-13 09:38:46,282] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 213 +26: [2023-05-13 09:38:46,288] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 213 +26: [2023-05-13 09:38:46,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt. +26: [2023-05-13 09:38:46,309] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 211 +26: [2023-05-13 09:38:46,318] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 211 +13: [2023-05-13 09:38:46,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt. +13: [2023-05-13 09:38:46,327] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 111 +13: [2023-05-13 09:38:46,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt. +13: [2023-05-13 09:38:46,327] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 107 +13: [2023-05-13 09:38:46,333] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 111 +13: [2023-05-13 09:38:46,334] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 107 +10: [2023-05-13 09:38:46,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:38:46,345] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 85 +10: [2023-05-13 09:38:46,352] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 85 +14: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt. +14: [2023-05-13 09:38:46,353] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 115 +12: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt. +12: [2023-05-13 09:38:46,354] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 101 +13: [2023-05-13 09:38:46,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt. +13: [2023-05-13 09:38:46,356] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 109 +26: [2023-05-13 09:38:46,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt. +26: [2023-05-13 09:38:46,357] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 209 +14: [2023-05-13 09:38:46,360] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 115 +10: [2023-05-13 09:38:46,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:38:46,361] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 81 +12: [2023-05-13 09:38:46,362] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 101 +13: [2023-05-13 09:38:46,363] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 109 +26: [2023-05-13 09:38:46,365] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 209 +27: [2023-05-13 09:38:46,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt. +27: [2023-05-13 09:38:46,366] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 219 +10: [2023-05-13 09:38:46,367] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 81 +26: [2023-05-13 09:38:46,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt. +26: [2023-05-13 09:38:46,369] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 215 +27: [2023-05-13 09:38:46,374] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 219 +26: [2023-05-13 09:38:46,376] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 215 +12: [2023-05-13 09:38:46,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt. +12: [2023-05-13 09:38:46,378] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 103 +25: [2023-05-13 09:38:46,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:38:46,384] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 103 +25: [2023-05-13 09:38:46,384] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 202 +25: [2023-05-13 09:38:46,390] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 202 +21: [2023-05-13 09:38:46,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:38:46,412] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 173 +21: [2023-05-13 09:38:46,418] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 173 + 1: [2023-05-13 09:38:46,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:38:46,421] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 9 +25: [2023-05-13 09:38:46,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt. +25: [2023-05-13 09:38:46,423] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 207 +21: [2023-05-13 09:38:46,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:38:46,427] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 169 + 1: [2023-05-13 09:38:46,427] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 9 +25: [2023-05-13 09:38:46,429] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 207 +21: [2023-05-13 09:38:46,434] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 169 +13: [2023-05-13 09:38:46,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt. +13: [2023-05-13 09:38:46,436] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 105 +11: [2023-05-13 09:38:46,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:38:46,444] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 95 +13: [2023-05-13 09:38:46,443] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 105 +16: [2023-05-13 09:38:46,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt. +16: [2023-05-13 09:38:46,447] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 135 +20: [2023-05-13 09:38:46,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt. +20: [2023-05-13 09:38:46,448] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 163 +11: [2023-05-13 09:38:46,450] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 95 +16: [2023-05-13 09:38:46,453] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 135 +20: [2023-05-13 09:38:46,455] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 163 +25: [2023-05-13 09:38:46,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt. +25: [2023-05-13 09:38:46,460] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 205 +11: [2023-05-13 09:38:46,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:38:46,462] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 89 +14: [2023-05-13 09:38:46,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt. +14: [2023-05-13 09:38:46,464] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 119 +25: [2023-05-13 09:38:46,467] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 205 +11: [2023-05-13 09:38:46,469] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 89 +14: [2023-05-13 09:38:46,471] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 119 +25: [2023-05-13 09:38:46,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt. +25: [2023-05-13 09:38:46,474] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 204 +17: [2023-05-13 09:38:46,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt. +17: [2023-05-13 09:38:46,478] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 137 +25: [2023-05-13 09:38:46,481] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 204 +31: [2023-05-13 09:38:46,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt. +31: [2023-05-13 09:38:46,481] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 255 +19: [2023-05-13 09:38:46,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt. +19: [2023-05-13 09:38:46,484] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 157 +17: [2023-05-13 09:38:46,484] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 137 +25: [2023-05-13 09:38:46,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt. +25: [2023-05-13 09:38:46,486] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 206 +31: [2023-05-13 09:38:46,487] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 255 +22: [2023-05-13 09:38:46,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt. +22: [2023-05-13 09:38:46,488] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 181 +19: [2023-05-13 09:38:46,490] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 157 +25: [2023-05-13 09:38:46,493] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 206 +21: [2023-05-13 09:38:46,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:38:46,492] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 171 +27: [2023-05-13 09:38:46,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt. +27: [2023-05-13 09:38:46,493] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 223 + 8: [2023-05-13 09:38:46,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt. +20: [2023-05-13 09:38:46,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt. + 8: [2023-05-13 09:38:46,493] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 65 +20: [2023-05-13 09:38:46,494] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 165 +22: [2023-05-13 09:38:46,495] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 181 +21: [2023-05-13 09:38:46,499] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 171 +27: [2023-05-13 09:38:46,500] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 223 +20: [2023-05-13 09:38:46,500] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 165 + 8: [2023-05-13 09:38:46,500] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 65 +19: [2023-05-13 09:38:46,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt. +20: [2023-05-13 09:38:46,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt. +16: [2023-05-13 09:38:46,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt. +19: [2023-05-13 09:38:46,505] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 159 +20: [2023-05-13 09:38:46,505] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 161 +16: [2023-05-13 09:38:46,505] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 131 +10: [2023-05-13 09:38:46,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:38:46,506] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 87 +20: [2023-05-13 09:38:46,511] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 161 +19: [2023-05-13 09:38:46,512] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 159 +16: [2023-05-13 09:38:46,511] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 131 +17: [2023-05-13 09:38:46,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt. +17: [2023-05-13 09:38:46,514] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 139 +27: [2023-05-13 09:38:46,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:38:46,514] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 87 +27: [2023-05-13 09:38:46,515] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 221 +17: [2023-05-13 09:38:46,521] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 139 +27: [2023-05-13 09:38:46,522] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 221 +16: [2023-05-13 09:38:46,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt. +16: [2023-05-13 09:38:46,526] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 133 + 2: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:38:46,529] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 19 +23: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:38:46,531] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 185 + 8: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt. +16: [2023-05-13 09:38:46,532] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 133 + 8: [2023-05-13 09:38:46,532] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 71 +24: [2023-05-13 09:38:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt. +24: [2023-05-13 09:38:46,533] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 197 +18: [2023-05-13 09:38:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:38:46,534] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 149 + 2: [2023-05-13 09:38:46,536] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 19 +18: [2023-05-13 09:38:46,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:38:46,537] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 151 +24: [2023-05-13 09:38:46,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt. +24: [2023-05-13 09:38:46,538] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 195 + 8: [2023-05-13 09:38:46,538] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 71 +23: [2023-05-13 09:38:46,539] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 185 +24: [2023-05-13 09:38:46,540] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 197 +18: [2023-05-13 09:38:46,541] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 149 +16: [2023-05-13 09:38:46,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:38:46,543] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 151 +16: [2023-05-13 09:38:46,544] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 129 +24: [2023-05-13 09:38:46,545] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 195 +24: [2023-05-13 09:38:46,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt. +24: [2023-05-13 09:38:46,549] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 193 +16: [2023-05-13 09:38:46,551] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 129 + 6: [2023-05-13 09:38:46,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt. + 6: [2023-05-13 09:38:46,553] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 51 +24: [2023-05-13 09:38:46,555] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 193 + 6: [2023-05-13 09:38:46,559] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 51 +28: [2023-05-13 09:38:46,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt. +28: [2023-05-13 09:38:46,559] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 231 +11: [2023-05-13 09:38:46,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:38:46,561] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 91 +22: [2023-05-13 09:38:46,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt. +22: [2023-05-13 09:38:46,564] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 179 +28: [2023-05-13 09:38:46,565] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 231 +11: [2023-05-13 09:38:46,568] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 91 +22: [2023-05-13 09:38:46,572] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 179 + 6: [2023-05-13 09:38:46,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt. + 6: [2023-05-13 09:38:46,572] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 55 +18: [2023-05-13 09:38:46,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:38:46,573] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 145 + 6: [2023-05-13 09:38:46,579] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 55 +18: [2023-05-13 09:38:46,579] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 145 +17: [2023-05-13 09:38:46,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt. +17: [2023-05-13 09:38:46,585] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 143 +11: [2023-05-13 09:38:46,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:38:46,586] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 93 +17: [2023-05-13 09:38:46,592] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 143 +11: [2023-05-13 09:38:46,592] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 93 + 3: [2023-05-13 09:38:46,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt. + 3: [2023-05-13 09:38:46,603] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 31 +10: [2023-05-13 09:38:46,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:38:46,608] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 83 + 3: [2023-05-13 09:38:46,609] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 31 +24: [2023-05-13 09:38:46,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt. +24: [2023-05-13 09:38:46,612] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 199 +19: [2023-05-13 09:38:46,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt. +19: [2023-05-13 09:38:46,614] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 153 +10: [2023-05-13 09:38:46,616] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 83 + 9: [2023-05-13 09:38:46,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt. + 9: [2023-05-13 09:38:46,618] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 79 +24: [2023-05-13 09:38:46,618] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 199 +19: [2023-05-13 09:38:46,622] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 153 + 9: [2023-05-13 09:38:46,624] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 79 + 5: [2023-05-13 09:38:46,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:38:46,635] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 43 +19: [2023-05-13 09:38:46,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt. +19: [2023-05-13 09:38:46,638] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 155 +15: [2023-05-13 09:38:46,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt. +15: [2023-05-13 09:38:46,640] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 123 + 5: [2023-05-13 09:38:46,641] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 43 + 1: [2023-05-13 09:38:46,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:38:46,643] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 15 + 8: [2023-05-13 09:38:46,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt. + 8: [2023-05-13 09:38:46,643] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 67 +19: [2023-05-13 09:38:46,645] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 155 +15: [2023-05-13 09:38:46,647] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 123 + 1: [2023-05-13 09:38:46,649] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 15 + 8: [2023-05-13 09:38:46,650] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 67 + 8: [2023-05-13 09:38:46,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:38:46,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt. + 8: [2023-05-13 09:38:46,651] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 69 + 2: [2023-05-13 09:38:46,651] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 21 + 2: [2023-05-13 09:38:46,658] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 21 + 8: [2023-05-13 09:38:46,658] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 69 + 7: [2023-05-13 09:38:46,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt. + 7: [2023-05-13 09:38:46,663] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 59 + 5: [2023-05-13 09:38:46,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:38:46,670] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 41 + 7: [2023-05-13 09:38:46,670] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 59 +30: [2023-05-13 09:38:46,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt. +30: [2023-05-13 09:38:46,677] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 247 + 5: [2023-05-13 09:38:46,677] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 41 +22: [2023-05-13 09:38:46,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt. +22: [2023-05-13 09:38:46,680] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 183 +18: [2023-05-13 09:38:46,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:38:46,683] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 147 +30: [2023-05-13 09:38:46,683] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 247 +22: [2023-05-13 09:38:46,686] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 183 +18: [2023-05-13 09:38:46,690] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 147 + 6: [2023-05-13 09:38:46,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt. + 6: [2023-05-13 09:38:46,696] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 49 + 6: [2023-05-13 09:38:46,702] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 49 +22: [2023-05-13 09:38:46,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt. +22: [2023-05-13 09:38:46,706] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 177 +22: [2023-05-13 09:38:46,716] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 177 + 2: [2023-05-13 09:38:46,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:38:46,721] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 23 + 5: [2023-05-13 09:38:46,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:38:46,723] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 45 + 2: [2023-05-13 09:38:46,727] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 23 +25: [2023-05-13 09:38:46,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:38:46,730] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 45 +25: [2023-05-13 09:38:46,731] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 203 + 6: [2023-05-13 09:38:46,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt. + 6: [2023-05-13 09:38:46,735] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 53 +25: [2023-05-13 09:38:46,738] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 203 + 1: [2023-05-13 09:38:46,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:38:46,740] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 13 + 6: [2023-05-13 09:38:46,742] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 53 +30: [2023-05-13 09:38:46,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt. +30: [2023-05-13 09:38:46,745] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 243 + 1: [2023-05-13 09:38:46,746] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 13 +30: [2023-05-13 09:38:46,751] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 243 +30: [2023-05-13 09:38:46,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt. +30: [2023-05-13 09:38:46,771] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 241 +30: [2023-05-13 09:38:46,778] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 241 + 9: [2023-05-13 09:38:46,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt. + 9: [2023-05-13 09:38:46,784] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 73 +23: [2023-05-13 09:38:46,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:38:46,786] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 189 + 9: [2023-05-13 09:38:46,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt. + 9: [2023-05-13 09:38:46,786] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 77 + 9: [2023-05-13 09:38:46,791] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 73 + 9: [2023-05-13 09:38:46,793] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 77 +23: [2023-05-13 09:38:46,793] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 189 + 7: [2023-05-13 09:38:46,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt. + 7: [2023-05-13 09:38:46,799] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 57 + 7: [2023-05-13 09:38:46,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt. + 7: [2023-05-13 09:38:46,800] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 63 + 7: [2023-05-13 09:38:46,806] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 57 + 7: [2023-05-13 09:38:46,807] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 63 + 5: [2023-05-13 09:38:46,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:38:46,837] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 47 +17: [2023-05-13 09:38:46,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt. +17: [2023-05-13 09:38:46,841] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 141 + 5: [2023-05-13 09:38:46,843] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 47 +15: [2023-05-13 09:38:46,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt. +15: [2023-05-13 09:38:46,845] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 125 +17: [2023-05-13 09:38:46,848] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 141 +30: [2023-05-13 09:38:46,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt. +30: [2023-05-13 09:38:46,849] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 245 +15: [2023-05-13 09:38:46,851] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 125 +30: [2023-05-13 09:38:46,857] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 245 +25: [2023-05-13 09:38:46,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt. +25: [2023-05-13 09:38:46,870] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 201 + 7: [2023-05-13 09:38:46,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt. + 7: [2023-05-13 09:38:46,871] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 61 +25: [2023-05-13 09:38:46,877] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 201 + 7: [2023-05-13 09:38:46,877] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 61 +15: [2023-05-13 09:38:46,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt. +15: [2023-05-13 09:38:46,889] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 121 +23: [2023-05-13 09:38:46,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:38:46,892] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 187 +15: [2023-05-13 09:38:46,895] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 121 +23: [2023-05-13 09:38:46,899] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 187 +23: [2023-05-13 09:38:46,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:38:46,961] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 191 +23: [2023-05-13 09:38:46,967] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 191 +31: [2023-05-13 09:38:47,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt. +31: [2023-05-13 09:38:47,038] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 251 +31: [2023-05-13 09:38:47,044] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 251 + 1: [2023-05-13 09:38:47,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:38:47,055] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 11 + 1: [2023-05-13 09:38:47,061] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 11 +15: [2023-05-13 09:38:47,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt. +15: [2023-05-13 09:38:47,473] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 127 +15: [2023-05-13 09:38:47,480] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 127 +21: [2023-05-13 09:38:47,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:38:47,633] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 175 +21: [2023-05-13 09:38:47,641] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 175 + 0: [2023-05-13 09:38:48,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. + 0: [2023-05-13 09:38:48,296] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 4 + 0: [2023-05-13 09:38:48,302] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 4 +31: [2023-05-13 09:38:48,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt. +31: [2023-05-13 09:38:48,391] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 253 +31: [2023-05-13 09:38:48,396] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 253 +12: [2023-05-13 09:38:48,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:38:48,440] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 100 +12: [2023-05-13 09:38:48,449] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 100 +31: [2023-05-13 09:38:48,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt. +31: [2023-05-13 09:38:48,468] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 249 +31: [2023-05-13 09:38:48,474] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 249 + 0: [2023-05-13 09:38:48,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. + 0: [2023-05-13 09:38:48,641] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 5 + 0: [2023-05-13 09:38:48,648] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 5 + 0: [2023-05-13 09:38:49,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. + 0: [2023-05-13 09:38:49,634] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 3 + 0: [2023-05-13 09:38:49,641] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 3 + 0: [2023-05-13 09:38:50,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. + 0: [2023-05-13 09:38:50,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. + 0: [2023-05-13 09:38:50,043] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 2 + 0: [2023-05-13 09:38:50,043] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 7 + 0: [2023-05-13 09:38:50,050] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 2 + 0: [2023-05-13 09:38:50,050] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 7 + 9: [2023-05-13 09:38:50,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt. + 9: [2023-05-13 09:38:50,570] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 75 + 9: [2023-05-13 09:38:50,580] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 75 + 0: [2023-05-13 09:38:51,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. + 0: [2023-05-13 09:38:51,542] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 1 + 0: [2023-05-13 09:38:51,550] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 1 + 0: successfully loaded checkpoint from checkpoints_4b284bc4perplexity at iteration 0 +31: time (ms) | load-checkpoint: 23590.87 + 0: estimated model parameters: 4.24648704 + 0: estimated model parameters without embeddings: 4.078977024 + 0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-05-13 09:38:52 + 0: > building train, validation, and test datasets ... + 0: > datasets target sizes (minimum size): + 0: train: 1 + 0: validation: 102400 + 0: test: 102400 + 0: > building train, validation, and test datasets for GPT ... + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.007684 seconds + 0: number of documents: 3133972 + 0: > dataset split: + 0: train: + 0: document indices in [0, 3133972) total of 3133972 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.040 seconds + 0: total number of samples: 731002 + 0: total number of epochs: 1 + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.005663 seconds + 0: number of documents: 364608 + 0: > dataset split: + 0: validation: + 0: document indices in [0, 364608) total of 364608 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.014 seconds + 0: total number of samples: 169955 + 0: total number of epochs: 2 + 0: > finished creating GPT datasets ... + 0: [after dataloaders are built] datetime: 2023-05-13 09:38:58 + 0: done with setup ... + 0: training ... +31: time (ms) | model-and-optimizer-setup: 39591.22 | train/valid/test-data-iterators-setup: 5984.03 + 0: [after training is done] datetime: 2023-05-13 09:38:58 +31: ----------------------------------------------------------------------------------------------------------------- +31: validation loss at the end of training for val data | lm loss value: 2.496036E+00 | lm loss PPL: 1.213430E+01 | +31: ----------------------------------------------------------------------------------------------------------------- +END 3510897: Sat 13 May 2023 09:41:07 AM EEST diff --git a/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_0.json b/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..c26cde30fad4e10d7c6fa319e1a1facc1b225663 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.30438087878461806, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.027985398209556842}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.06571872727613429, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014178747302171335}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.28999616131453776, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00434042822819289}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.10072645925520833, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001803334386023207}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.030429296094824455, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008978277172003451}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.13772743526874254, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0029713701289473594}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.046598784604859486, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011368379760255077}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.06388836453568124, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013632881297082904}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.28363499948165843, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004278943131293478}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.09804723740892954, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001738122594593388}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.06323531180120187, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013639142517225444}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.2788262564249966, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004155727073631924}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.09688278752987489, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017265830325622752}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_1.json b/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..1d62e0856b6dd46de886a07c2dd311356cab0e14 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.3677620101657523, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02209751845324868}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.06882276794192525, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001269726775890253}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.3358838633226288, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004944830474767479}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.10743506199331603, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017479884253783432}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.03198910527175372, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007904311483238523}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.1639727133230724, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0035115320297904157}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.050065381446004406, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011139596034513608}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.06619223938605175, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001213536084420925}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.3209215286944476, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004642641539916205}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.10330309525862182, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016709978496468228}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.06596557560336991, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012138292029559524}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.32099921969419465, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004628086148730377}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.102932245838326, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016665533221243655}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_2.json b/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..c1a4aef00621eda52c5027de35e281c6113739da --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.47923990514244874, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03166485764907936}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07204383319731683, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0012155881365152256}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.36672271781887156, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00504889217499601}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.11370260047327577, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017046442313663663}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.03340955873075625, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007555261485064188}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.18177633862979822, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0037021276711014387}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05299300887318277, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010947490193883183}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.0684099274474328, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00113914940883586}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.3455577571113544, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004627024307619622}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.1079044075881634, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015944656814960549}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.06889283297975032, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011627655720725584}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.3493606357170359, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0047163037850670355}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.10867383319593066, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016268811944244747}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_3.json b/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9d734eb27d80376cc9c40d5c9a1f4006fbb4729c --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.5027920774081681, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.030462214372567638}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07255116336657369, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0011977538039791803}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.38345627514615743, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.005203246900271579}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.1146798040803432, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016732637014203531}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.03363690204181306, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007452192351280848}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.1918451527107662, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0038340869615008937}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05346240993388053, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00108055575928216}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.06838424060537425, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011261261420472187}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.35815409259444997, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004736318677371466}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.10795694469281018, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015660316693302382}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.06920483129615293, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011443226072995456}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.36357801556888564, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004808733759405405}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.10928613566777898, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015925978011445418}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_4.json b/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..71ecf992b3c65d9185e35d3ebdb5656d89f15fc1 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.5831555359984453, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.032234659529391536}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07392661038451319, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00117243887617427}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.39335666383917167, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.005142725185391461}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.11750008800672353, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016477505264978866}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.03448655846022897, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000729028189759837}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.20020642249358464, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0038687044782014456}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05523835531458514, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010698113777651895}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.06941307599352699, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0010899673683375645}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.3670211800134988, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004700921629116034}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.1101991356020512, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015236328659278776}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07037653651757456, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011110620728147437}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.3732246975137694, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004797544529790582}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.1117873702364502, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015580968509773768}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_5.json b/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..d15130e1c88d45cad450b0591f7717074fbbf6c7 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.6365749188582466, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.039638813172299045}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07471557189793104, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001168900413573336}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.40490074663891484, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.005284549026027036}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.11886210839837029, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001641331021319136}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.0350998165096206, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007323935879922631}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.20683097072034598, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003912349890143833}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05619659179665226, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010637830284484662}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.06926051144904892, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0010863629617073184}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.37213644980404864, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004688196830703624}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.10996565063664662, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015072851492445695}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07111141425693926, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011190880115330949}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.38316893134013263, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004892613100508629}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.11299990359140699, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001561286385930656}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_0.json b/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..39293342f2ca6684ee410ca41c46f65fb082d71c --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.14888065220459862, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018522381431221608}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.2533502341723187, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002694764028887335}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.17406988949643304, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018339930055680368}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.029169878872080838, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007679053115951254}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.05196743771903658, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001421253503809284}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.03434823579340285, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008465750128095288}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.1138889494347587, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013014428116691651}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.20025749658243655, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021830666467128335}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.13456472416719728, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013210328093623985}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.13762861506852897, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001700786002748469}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.23544002194089206, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002515653712672603}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.16115176655656072, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016863623652902142}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.5579291268345696, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06016300187830401}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_1.json b/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..007c0a362a7c00b3b721a182c7e2b6eb34ee722b --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.18628138897984436, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0021004152459147747}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.3129007187114192, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0029032182486249418}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.21615787975223222, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001980052888687319}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.046486537780770956, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009461054630336336}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.08171808437787392, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001737593596442716}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.05439999185109123, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001031487571403725}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.13401183830304364, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00140969758282127}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.23246416864486025, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022971477521400944}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.15690618601267947, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013436606219927435}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.1743604474408626, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001958253968449097}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.2946260029722949, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0027768807102536105}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.20269374476846683, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018536229956474099}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 2.8504691019972226, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07529997127562747}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_2.json b/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..7d99c0282e8ea19533070878271ccd4afc83bbad --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.19145535496087807, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002296451033052049}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.2975680142091921, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002931688880420891}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.21248721018427344, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002005377613049306}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.05003016577994891, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0011239070261388475}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.08044311392175647, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001763848842609043}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.055330208196910265, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010696008344144292}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.14417467914561882, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0016764997188872376}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.23103867510190504, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002419663302485}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.1613056472114288, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014427201039189429}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.17923916059988207, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002160531408137847}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.27981692016794185, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002789997688790343}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.19917102006805654, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018806834912790124}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 3.127885213179027, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05567768230878822}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_3.json b/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..cdb083da8e2970d5159e6c1604b70d89401d4b77 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.17616795901868026, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0029041735231821264}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.2398365427132762, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003303934643917804}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.1775686933392586, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0023132433314061317}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.046939802020617566, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0014687470735497206}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.06422548105107985, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016475717099470786}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.046193370406017856, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010778055491266162}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.13481464964787096, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0022897360632217266}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.18678109287740904, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0026738452831759677}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.135715301519862, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017310055665623648}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.1658688389623912, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002743944725824659}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.22654605759305751, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0031428598277916196}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.1672145914904344, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021753093573102073}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 3.0807771429420754, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.09283762605369977}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_4.json b/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..fb813f8869dbd7514f069d1ba654d7f048c19819 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.0584713690765183, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0023318063534537165}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.07718352458545524, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002735992768065453}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.05573280124002876, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019136745631413645}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.01500160633710272, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009825454465574395}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.02039516990198183, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001139121615096021}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.013946881394381343, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007038540032034839}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.04558030719768415, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0018674171866451237}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.06093944803992283, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022091397429612157}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.04301362739459154, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014516622611586488}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.05496664162339461, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022050347910565274}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.0725975299250868, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025818847671325463}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.052282734877528686, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017926957831151907}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.5293847081914334, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.044698835511384254}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_5.json b/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..96a7bb0233de6f3b500c60868d3d8d1c62d5d2fe --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.009305181256700445, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0010395448742730087}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.012223369402839458, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0012274307651446278}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.008623575125031323, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0008288612961403041}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.0025435206455146683, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005383342373842493}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.0031006973107968873, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0004472222802474402}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.002093015608191419, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00027758841441824164}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.007394254420247825, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.000882581070383217}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.009645269299470693, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0009987275588991668}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.006710465163509879, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0006498829914466451}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.008824151605454963, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0010021873301601862}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.011594578158050393, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0011725427133673896}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.008136271016090088, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0007837539033952645}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.9499255251335006e-07, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 3.3145291785030997e-07}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_0.json b/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..2dc95757fa264b223b3aa290242a2d2c7524bc1c --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 1.82061032660784, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07096469051250175}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.222486766832112, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0033842158110189277}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.1790689833398624, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0018970549220355723}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.15825083088882455, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0015057952646897993}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.021935233358876982, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008022922500453108}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.031732352695645706, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0009741385489925919}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.022292985524574444, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0006610427910211585}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.20561351860512828, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0030700557880713867}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.17239931551868604, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.001867012935877257}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.1495036471572004, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013950962210181237}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.19805184360305841, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0032227599715627343}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.1509451752386678, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0016062725531267121}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.136862319950687, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0014225741473040676}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_1.json b/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..55f68e18169122c1c964d72efe16f33a37ce9abe --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 11.657037239702142, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1250980356694754}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.45968566711122216, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002881285091661702}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.46542006430779354, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0028779976054145785}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4389348120897716, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002063579737266741}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.21038656753149504, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002167887993557998}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.21239637096024527, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0021227522877282565}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.1995055031643681, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017838055449605266}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3268614579954877, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002404585448478252}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.3303654420846652, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002362681762163533}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3114176922977353, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017952929940337735}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.3793916638196591, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002677199223367296}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.3835618439741999, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002671098460743732}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3620752417019191, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002056079671135789}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_2.json b/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e99dc449d157b817f2fb9c6eee3259abcf3ee70f --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 14.339702198028428, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13350232957464767}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.5570264884781344, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0033490944418358374}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.46432809954638865, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002945932627099228}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4798241859576156, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022710941605897325}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.28313861767317927, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0027952153389596843}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.2325744710533778, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0022998468012822644}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.2403073697075928, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.002087685451857004}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.4135082878473266, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.003050078856711996}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.34252272942973466, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002524272588321217}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.35446236973706474, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0021166068311389095}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.46552847624479543, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0032616657748971998}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.3869495647860439, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002764146985697145}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.40022368564912997, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.00229931320451958}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_3.json b/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..954819b13ccbc52a8a540596186ab748d44bd453 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 14.50537794842067, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11584363484363577}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.5402112307108176, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003295524595806651}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.47732917178018114, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00294022910982499}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4791848298362283, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022007720607771426}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.2750068424415456, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002742749101788008}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.239901278700609, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0023557789869618896}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.24067531660097596, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.002087534005568858}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.40045561756040954, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002992395353090302}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.35170929339590973, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0025447548206976338}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.35365228129689674, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00209824048462177}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.45223409928062946, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0031770144742772127}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.3996467826064061, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002820396617171887}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.40103023450016195, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002275399298911374}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_4.json b/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..9823acf79922ea82c86a47ee0688eeeabb69bc9b --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 13.323972252459379, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.16742143852884073}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.4899803542811925, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003129903470372565}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.48706786873461216, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002870795532573222}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.46396091436304504, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021581635430039958}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.24359351406261168, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0025312411079664366}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.24067281188228995, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0023396690126969532}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.2283664329607444, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0020126317997385106}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3624069982870762, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0027711033582985124}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.3591682726211815, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0025140239206322313}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3421085000388542, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002013555794275305}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.4114748595185199, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0029677058156554}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.4100795333879036, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002820033316291732}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3900177854874722, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0022269574328580566}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_5.json b/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..83b06c8e558eee148c1ddcfd516e67154bafcd46 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 12.462777642040733, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13551486783004044}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.45865129639331476, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002686570307588201}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4938245489163575, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027958101448835645}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4559772721525502, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020283797559365376}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.22301236793840634, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002160872250109803}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.24081053155333462, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0022973862129338496}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.22088952705630202, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0018905153937119456}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3354962184605829, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0023253516829045725}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.3619962097662624, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002476618191262458}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3335077717084848, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018782606369312164}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.3861502626280554, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0025748401786830784}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.416513089237172, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002733652456992395}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.38415093197988986, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.00209136460819391}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_0.json b/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..f08f89c9826e79ac9cc9e4a2c684d667d7a4e014 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.15955334897495682, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002209067557750088}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.36217161593458225, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004440471846187096}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.21547497313031236, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002589604938536055}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.039876602220101885, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0013408657891869856}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.09373979355812566, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003020539395998483}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.05432411992031246, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017179496173428985}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.12161602298528985, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0016687848022286617}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.277756692596081, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003476817201900778}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.16451405093380472, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0019625362254798284}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.12390275070243846, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001821194882097903}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.2841653396002695, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003918981158721574}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.1678765700309988, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002214619502903411}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 2.282427388603868, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12070696540892288}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_1.json b/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..6ed62158a61cc84b0440b24f895bbac4a4dddde0 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.12010749072738239, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0019038452514628232}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.2963599272578747, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004492167665019704}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.16895420412623663, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002594837744965486}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.02342764378945351, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001019840391470285}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.0606775250012725, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002681867584351387}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0333977078622104, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00144862051823388}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.09436779371850286, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014414694335729383}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.2345277453603614, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003510304651172983}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.1329671503634145, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0019756316848368602}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.09564640396609536, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015463429453530608}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.23817260800506607, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0037920083635799903}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.1348677191851774, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021296361098744635}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.3775738726107656, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08168674122275797}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_2.json b/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..81ab144ef48e39826ce6af99343f0fb0154b34ca --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.13018508734953355, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0019680938941104625}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.3210711017407105, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004611340413923777}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.183152447945667, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0026661841419304156}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.028176744941516615, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001075491149082574}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.07240802386862864, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002820788158486654}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.04006840275451415, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0015228269131846886}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.10199545313071355, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014580281840802887}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.2531633381335362, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003556665780315997}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.14371195073214757, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0019879171616702124}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.10303479198876377, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016024271297654439}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.2565576177699366, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003974104626273052}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.14532030438832994, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002206439734918519}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.6436795917006544, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07345990657620402}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_3.json b/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..8334518847da2f40819363b65571b5467189f499 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.13017675194898692, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002198538499348046}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.30922236829508537, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004838699494510795}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.17881939665307162, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0027596350151271876}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.02760225846524049, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0011161638977048621}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.06881042595527115, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0028092871690327376}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.03850524479452543, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0015068733635772717}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.10014486228665212, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0016329144933538103}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.2399877011308486, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0037910983726926846}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.13794374325281775, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0020813614818383725}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.10223719953083442, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001759902279193926}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.24547016572644775, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004083522562686188}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.1407907014518307, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0022509491687602674}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.6496742062970557, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10044687108522192}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_4.json b/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..b34e1e76b71157f3bc9b05ad8c128c64fd55abb5 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.04149135621677754, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0026386378608042917}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.07750948792703138, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004477188850582034}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.049656632315842844, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002856336743990567}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.008961527810790616, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009890519095079862}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.017840955869101484, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016844183124156535}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.010989532389756859, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001029101618129444}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.03154250432068281, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020252442164337293}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.05909979811818957, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0034347562251169}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.0375802335882974, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002147921108027172}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.03303780727965199, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002118654737857532}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.06214884490779202, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003663125602489163}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.03944906528704634, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0022725008097993537}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.8321034229031437, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08945851965278599}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_5.json b/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..9388c6a914accbe3b77ea2c41882eb45e437f906 --- /dev/null +++ b/perplexity/evaluation/generation/agg.perplexity_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.003117323860723007, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0008837148810752774}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.0026242515972514074, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0007336964722264186}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.0028022236632419895, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0007863889401033125}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.0005615954214733349, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00024697557291310967}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.0004889607248097814, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00023161252427707105}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0005154358964062439, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00023513093079516412}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.002278284775437062, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0006480855590254838}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.0019397313262420287, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0005430737577124271}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.002054859718325845, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005764790804455003}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.0024716725936861457, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0007191187737619554}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.0020845273284634726, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0005920007456566033}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.002218076936186532, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0006333184346671743}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 3.1493567208404783e-39, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 1.4249121699400145e-34}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_0.jsonl b/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..41cd4b0c0d1cf6172c0d385e142c6e326e37dfc9 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d307c0ef6b856bab906eaf628e6abb37f8842388c4e55c95e77c1e086f0d6a1 +size 4135526 diff --git a/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_1.jsonl b/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f4553c4a3bd500696f006e4e4bc5851232178650 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e27833aee33387958bf2571697bd197a0c84c70316ac37e23799e8fa3c8a0077 +size 5098507 diff --git a/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_2.jsonl b/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a3e66c881d664c7b5337e9c467bb3a3a307aba8b --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f22e7c12193b746d70654c83dbf65f833453ac292b274b195d4f0af5ca9f56e8 +size 5994331 diff --git a/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_3.jsonl b/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ab8475c3de71b1cc2ab45a70fa343733fd02db0b --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79264b00c160f248974e92c0bf7ec94941e62dc3cbfed672ffdab9bbeb243f44 +size 6922711 diff --git a/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_4.jsonl b/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a452acc91eebe775061f1fd9c551a08959c75c6f --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e20035c2645e665327400637df107d64e7e4348535e14c2f3f4c29d27fe406 +size 7822876 diff --git a/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_5.jsonl b/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2fa9674deef2f459f62c347404d7e07ef27b01f4 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_GEM-web_nlg_en_PALM_prompt_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48b300c546c09d0fe19e191acfc000b90147254c732fd99ef4a045db2aac6ed8 +size 8738448 diff --git a/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_0.jsonl b/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5cb89a47c360b47774c8c0bf2bf15f6f4033a3d9 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4fe4eae64aa71bf2b01f36dc7451cfddc16620f07f35a1bbe64ab36d441c23b +size 7675186 diff --git a/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_1.jsonl b/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..51914973762fac815771e9688f260968d107b480 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5902c88cf7ed1e98d3dc2b138d17fedd69d3df2bdd63fff8136746b60ed15c87 +size 13299510 diff --git a/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_2.jsonl b/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..cb4f57d46c41ba69085421b14defa0d345358a35 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4502f52ef7440d17223e39168ccb14f7d152d5b5c54e7cc200c1b7722ff39676 +size 18867322 diff --git a/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_3.jsonl b/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..72d10272cb724a90069ec47492ec50045e664983 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd57f4decca08d6c401d04cf73be5e526aad4db61eb34642263812c67914cebe +size 24267907 diff --git a/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_4.jsonl b/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..bbe39798f0e0ef732179ef4c6877f6f1cbb9c386 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:137b82cba367896394e1cf999b8b1486f6fc5d9d1d788f8e5107f9d09e577855 +size 29449359 diff --git a/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_5.jsonl b/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..eee8bb407cd9d2bafd60234fd64937c672b36abf --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_GEM-wiki_lingua_en_tldr_en_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31de04d3beecd7021a68665ed32242b360ea1085e136f991e0f9d679a2de70be +size 34795948 diff --git a/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl b/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3e029245d2029714152c9b6366fe566a3d765d6c --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:677576187f22771e50b9ee2a5602062c45a16f54b3b2d1716dbe4799febbfb30 +size 4219750 diff --git a/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl b/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0f8433bf086f7c3b84c5fd9ee3c65f5145f08f86 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cf412242e1a391b79a5961708d53d0934bb56e0a6d08c3fa1c2e2f8d333f92d +size 5131942 diff --git a/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl b/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..066e5383c3e1927a3fe1ac39164b3776a9c09398 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69f8332ba677e67bfe1b2c2d3898c9951fb2a6af1bef8b74249e4d75fe6ed303 +size 6133690 diff --git a/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl b/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4a3b60eb72bb82d66b0a2188bd95025a0e296e82 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10b4bea2343a63e17f061f412c9d0a51790aba53c8402557bb4a69b7d25c77d5 +size 7245342 diff --git a/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl b/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..507971a394ddb8b050cd367a622f323dbf0bde2c --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92164fa46a074df910f0a325772edaaf1f0aba572c46308e2d5def04fd09665c +size 8374218 diff --git a/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl b/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..593910566c448e6562dec9e4bda34c011759665d --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6e7f309d92cbf4c1d8276d91ce49766c6de09d222b347499475331d6aaa9718 +size 9491233 diff --git a/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_0.jsonl b/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c82c8533dc8731b2704d49352efbdb7b005a02f6 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ec43b2031ba9c8d567309f45c56c0c46ed51027d502cc0dee77af1399b8c3a4 +size 2824705 diff --git a/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_1.jsonl b/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1841bbae19ec49c436847b7f818472b9bec48330 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a64d676949dd7afa30c0aaccbc5730f42281aff561d355a6995017530709eae5 +size 5109143 diff --git a/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_2.jsonl b/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fd14f184aff3bd0695a910ee083b0fa50fab3559 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e875e269f2e5409ac1b1c7df236b872590f9a46a9fbdc87a1d5dbf808d39dfae +size 7379320 diff --git a/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_3.jsonl b/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5bd5e26eb479d4f10a8cfc85266c2f737bdd0250 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8421fb510ab1a959d44b86958db2f65249a3b9f96b4fb67f45be1ac90ed019d +size 9649100 diff --git a/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_4.jsonl b/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4d4762355db7e6ea85cb7d30426014f16210cc55 --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c6644ea596727ebc2b42e2faa1e0d86e31d24aa0e0b51aa25003ec1dcc69f6b +size 11672533 diff --git a/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_5.jsonl b/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1a2b9cdd9ce04a9775025f40f7f41f4c62d3a36d --- /dev/null +++ b/perplexity/evaluation/generation/examples.perplexity_gem_xsum_article_DOC_summary_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79fd24ce044e4f66ddebec23ddff4d4b1d36affc674239b1bf3b422bfd096690 +size 13897604 diff --git a/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_0.json b/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..f29b256e68f6823668c405d5639376a6d501972a --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.30438087878461806, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.027985398209556842 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.06571872727613429, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0014178747302171335 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.28999616131453776, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.00434042822819289 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.10072645925520833, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001803334386023207 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.030429296094824455, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0008978277172003451 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.13772743526874254, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0029713701289473594 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.046598784604859486, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011368379760255077 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.06388836453568124, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0013632881297082904 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.28363499948165843, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004278943131293478 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.09804723740892954, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001738122594593388 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.06323531180120187, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0013639142517225444 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.2788262564249966, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004155727073631924 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.09688278752987489, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0017265830325622752 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_1.json b/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..fb2678dace1f8f5e504b37f8d40dbb465fa0758c --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.3677620101657523, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.02209751845324868 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.06882276794192525, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001269726775890253 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.3358838633226288, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004944830474767479 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.10743506199331603, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0017479884253783432 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.03198910527175372, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007904311483238523 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.1639727133230724, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0035115320297904157 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.050065381446004406, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011139596034513608 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.06619223938605175, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.001213536084420925 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.3209215286944476, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004642641539916205 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.10330309525862182, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0016709978496468228 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.06596557560336991, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012138292029559524 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.32099921969419465, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004628086148730377 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.102932245838326, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016665533221243655 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_2.json b/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e76d5d9a3d93f0a0c18251ffb99f6d0141b6ad56 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.47923990514244874, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.03166485764907936 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07204383319731683, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0012155881365152256 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.36672271781887156, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.00504889217499601 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.11370260047327577, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0017046442313663663 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.03340955873075625, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007555261485064188 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.18177633862979822, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0037021276711014387 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05299300887318277, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010947490193883183 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.0684099274474328, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.00113914940883586 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.3455577571113544, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004627024307619622 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.1079044075881634, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015944656814960549 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.06889283297975032, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0011627655720725584 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.3493606357170359, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0047163037850670355 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.10867383319593066, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016268811944244747 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_3.json b/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..f7ebbeb62b73c5907e7d984c0aaa136eee05ecab --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.5027920774081681, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.030462214372567638 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07255116336657369, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0011977538039791803 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.38345627514615743, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.005203246900271579 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.1146798040803432, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0016732637014203531 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.03363690204181306, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007452192351280848 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.1918451527107662, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0038340869615008937 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05346240993388053, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00108055575928216 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.06838424060537425, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011261261420472187 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.35815409259444997, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004736318677371466 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.10795694469281018, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015660316693302382 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.06920483129615293, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0011443226072995456 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.36357801556888564, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004808733759405405 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.10928613566777898, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015925978011445418 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_4.json b/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..be3f9a233577bf1764eb8456128b20f6a57a1a1f --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.5831555359984453, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.032234659529391536 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07392661038451319, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.00117243887617427 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.39335666383917167, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.005142725185391461 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.11750008800672353, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0016477505264978866 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.03448655846022897, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.000729028189759837 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.20020642249358464, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0038687044782014456 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05523835531458514, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010698113777651895 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.06941307599352699, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0010899673683375645 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.3670211800134988, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004700921629116034 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.1101991356020512, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015236328659278776 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07037653651757456, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0011110620728147437 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.3732246975137694, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004797544529790582 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.1117873702364502, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015580968509773768 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_5.json b/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..034c707df927b3c13df0200a31c388af1ddb6def --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.6365749188582466, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.039638813172299045 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07471557189793104, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001168900413573336 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.40490074663891484, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.005284549026027036 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.11886210839837029, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001641331021319136 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.0350998165096206, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007323935879922631 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.20683097072034598, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003912349890143833 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05619659179665226, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010637830284484662 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.06926051144904892, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0010863629617073184 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.37213644980404864, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004688196830703624 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.10996565063664662, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015072851492445695 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07111141425693926, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0011190880115330949 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.38316893134013263, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004892613100508629 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.11299990359140699, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001561286385930656 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_0.json b/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..10eab1f48d678f3e7bfbf1cecb92518c293195b2 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.14888065220459862, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0018522381431221608 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.2533502341723187, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.002694764028887335 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.17406988949643304, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0018339930055680368 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.029169878872080838, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007679053115951254 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.05196743771903658, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.001421253503809284 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.03434823579340285, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0008465750128095288 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.1138889494347587, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0013014428116691651 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.20025749658243655, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0021830666467128335 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.13456472416719728, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0013210328093623985 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.13762861506852897, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001700786002748469 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.23544002194089206, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.002515653712672603 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.16115176655656072, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016863623652902142 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.5579291268345696, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.06016300187830401 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_1.json b/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..59229a544abd05f5c19fb4fe12c68899e4c64255 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.18628138897984436, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0021004152459147747 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.3129007187114192, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0029032182486249418 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.21615787975223222, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001980052888687319 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.046486537780770956, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0009461054630336336 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.08171808437787392, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.001737593596442716 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.05439999185109123, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.001031487571403725 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.13401183830304364, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.00140969758282127 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.23246416864486025, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0022971477521400944 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.15690618601267947, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0013436606219927435 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.1743604474408626, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001958253968449097 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.2946260029722949, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0027768807102536105 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.20269374476846683, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018536229956474099 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 2.8504691019972226, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.07529997127562747 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_2.json b/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..2fc4ffee8d6aed493facdfb39c51c944e53bbe52 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.19145535496087807, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.002296451033052049 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.2975680142091921, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.002931688880420891 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.21248721018427344, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.002005377613049306 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.05003016577994891, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0011239070261388475 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.08044311392175647, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.001763848842609043 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.055330208196910265, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010696008344144292 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.14417467914561882, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0016764997188872376 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.23103867510190504, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.002419663302485 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.1613056472114288, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0014427201039189429 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.17923916059988207, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.002160531408137847 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.27981692016794185, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.002789997688790343 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.19917102006805654, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018806834912790124 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 3.127885213179027, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.05567768230878822 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_3.json b/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..4f0e2054fc55e13b435b34d7ac7fd8e32bbfb0c0 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.17616795901868026, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0029041735231821264 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.2398365427132762, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.003303934643917804 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.1775686933392586, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0023132433314061317 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.046939802020617566, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0014687470735497206 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.06422548105107985, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0016475717099470786 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.046193370406017856, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010778055491266162 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.13481464964787096, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0022897360632217266 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.18678109287740904, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0026738452831759677 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.135715301519862, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0017310055665623648 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.1658688389623912, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.002743944725824659 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.22654605759305751, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0031428598277916196 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.1672145914904344, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0021753093573102073 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 3.0807771429420754, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.09283762605369977 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_4.json b/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c6d2851b477d354855ab8dca47dbc7db9eb45110 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.0584713690765183, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0023318063534537165 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.07718352458545524, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.002735992768065453 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.05573280124002876, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0019136745631413645 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.01500160633710272, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0009825454465574395 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.02039516990198183, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.001139121615096021 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.013946881394381343, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0007038540032034839 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.04558030719768415, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0018674171866451237 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.06093944803992283, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0022091397429612157 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.04301362739459154, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0014516622611586488 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.05496664162339461, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0022050347910565274 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.0725975299250868, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0025818847671325463 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.052282734877528686, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0017926957831151907 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.5293847081914334, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.044698835511384254 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_5.json b/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..f590824dff9fd6c50363470100b80db3af79a986 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.009305181256700445, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0010395448742730087 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.012223369402839458, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0012274307651446278 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.008623575125031323, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0008288612961403041 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.0025435206455146683, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0005383342373842493 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.0031006973107968873, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0004472222802474402 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.002093015608191419, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00027758841441824164 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.007394254420247825, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.000882581070383217 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.009645269299470693, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0009987275588991668 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.006710465163509879, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0006498829914466451 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.008824151605454963, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0010021873301601862 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.011594578158050393, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0011725427133673896 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.008136271016090088, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0007837539033952645 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.9499255251335006e-07, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 3.3145291785030997e-07 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_0.json b/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ed1975c4c969d797bf5950429373e87343d4d597 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 1.82061032660784, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.07096469051250175 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.222486766832112, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0033842158110189277 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.1790689833398624, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0018970549220355723 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.15825083088882455, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0015057952646897993 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.021935233358876982, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0008022922500453108 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.031732352695645706, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0009741385489925919 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.022292985524574444, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0006610427910211585 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.20561351860512828, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0030700557880713867 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.17239931551868604, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.001867012935877257 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.1495036471572004, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0013950962210181237 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.19805184360305841, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0032227599715627343 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.1509451752386678, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0016062725531267121 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.136862319950687, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0014225741473040676 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_1.json b/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..9bf9f26a35c5b4585cc76c77814c5694dee43be5 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 11.657037239702142, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.1250980356694754 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.45968566711122216, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.002881285091661702 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.46542006430779354, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0028779976054145785 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4389348120897716, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002063579737266741 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.21038656753149504, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002167887993557998 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.21239637096024527, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0021227522877282565 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.1995055031643681, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0017838055449605266 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3268614579954877, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002404585448478252 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.3303654420846652, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002362681762163533 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3114176922977353, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0017952929940337735 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.3793916638196591, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.002677199223367296 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.3835618439741999, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002671098460743732 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3620752417019191, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002056079671135789 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_2.json b/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..42ea34ad8e1e967e1de9c717e1f248d6f01b89d9 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 14.339702198028428, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.13350232957464767 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.5570264884781344, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0033490944418358374 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.46432809954638865, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002945932627099228 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4798241859576156, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0022710941605897325 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.28313861767317927, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0027952153389596843 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.2325744710533778, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0022998468012822644 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.2403073697075928, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.002087685451857004 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.4135082878473266, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.003050078856711996 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.34252272942973466, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002524272588321217 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.35446236973706474, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0021166068311389095 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.46552847624479543, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0032616657748971998 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.3869495647860439, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002764146985697145 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.40022368564912997, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.00229931320451958 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_3.json b/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..cbf26794f7b848b66c1e798db3f5c4242828918b --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 14.50537794842067, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.11584363484363577 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.5402112307108176, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003295524595806651 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.47732917178018114, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.00294022910982499 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4791848298362283, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0022007720607771426 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.2750068424415456, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002742749101788008 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.239901278700609, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0023557789869618896 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.24067531660097596, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.002087534005568858 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.40045561756040954, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002992395353090302 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.35170929339590973, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0025447548206976338 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.35365228129689674, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.00209824048462177 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.45223409928062946, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0031770144742772127 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.3996467826064061, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002820396617171887 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.40103023450016195, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002275399298911374 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_4.json b/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..fff20523119adbff544624ac0deb675f2f50ed6f --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 13.323972252459379, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.16742143852884073 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.4899803542811925, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003129903470372565 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.48706786873461216, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002870795532573222 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.46396091436304504, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0021581635430039958 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.24359351406261168, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0025312411079664366 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.24067281188228995, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0023396690126969532 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.2283664329607444, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0020126317997385106 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3624069982870762, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0027711033582985124 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.3591682726211815, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0025140239206322313 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3421085000388542, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.002013555794275305 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.4114748595185199, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0029677058156554 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.4100795333879036, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002820033316291732 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3900177854874722, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0022269574328580566 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_5.json b/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..239bc8471164d4394056d30dba29d60b953f3b2a --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 12.462777642040733, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.13551486783004044 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.45865129639331476, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.002686570307588201 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4938245489163575, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0027958101448835645 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4559772721525502, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0020283797559365376 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.22301236793840634, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002160872250109803 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.24081053155333462, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0022973862129338496 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.22088952705630202, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0018905153937119456 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3354962184605829, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0023253516829045725 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.3619962097662624, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002476618191262458 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3335077717084848, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0018782606369312164 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.3861502626280554, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0025748401786830784 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.416513089237172, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002733652456992395 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.38415093197988986, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.00209136460819391 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_0.json b/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..9541f730264c1f2359737d1e0489cada67566856 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.15955334897495682, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.002209067557750088 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.36217161593458225, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004440471846187096 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.21547497313031236, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002589604938536055 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.039876602220101885, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0013408657891869856 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.09373979355812566, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.003020539395998483 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.05432411992031246, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0017179496173428985 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.12161602298528985, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0016687848022286617 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.277756692596081, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.003476817201900778 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.16451405093380472, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0019625362254798284 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.12390275070243846, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.001821194882097903 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.2841653396002695, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.003918981158721574 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.1678765700309988, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.002214619502903411 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 2.282427388603868, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.12070696540892288 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_1.json b/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..c492bf44d83f5c62df118c5a1f18767e2fc49fac --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.12010749072738239, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0019038452514628232 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.2963599272578747, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004492167665019704 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.16895420412623663, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002594837744965486 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.02342764378945351, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.001019840391470285 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.0606775250012725, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.002681867584351387 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.0333977078622104, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.00144862051823388 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.09436779371850286, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0014414694335729383 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.2345277453603614, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.003510304651172983 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.1329671503634145, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0019756316848368602 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.09564640396609536, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0015463429453530608 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.23817260800506607, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0037920083635799903 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.1348677191851774, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0021296361098744635 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.3775738726107656, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.08168674122275797 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_2.json b/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..8ae280e7011542a25bf99a72018f6c3b2c3f445a --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.13018508734953355, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0019680938941104625 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.3210711017407105, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004611340413923777 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.183152447945667, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0026661841419304156 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.028176744941516615, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.001075491149082574 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.07240802386862864, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.002820788158486654 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.04006840275451415, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0015228269131846886 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.10199545313071355, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0014580281840802887 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.2531633381335362, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.003556665780315997 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.14371195073214757, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0019879171616702124 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.10303479198876377, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0016024271297654439 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.2565576177699366, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.003974104626273052 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.14532030438832994, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.002206439734918519 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.6436795917006544, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.07345990657620402 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_3.json b/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..87b5be2e95b67d3b860456f170467040c6ea4969 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.13017675194898692, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.002198538499348046 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.30922236829508537, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004838699494510795 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.17881939665307162, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0027596350151271876 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.02760225846524049, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0011161638977048621 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.06881042595527115, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0028092871690327376 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.03850524479452543, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0015068733635772717 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.10014486228665212, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0016329144933538103 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.2399877011308486, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0037910983726926846 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.13794374325281775, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0020813614818383725 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.10223719953083442, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.001759902279193926 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.24547016572644775, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.004083522562686188 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.1407907014518307, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0022509491687602674 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.6496742062970557, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.10044687108522192 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_4.json b/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..b68bb8dfc7da3d2ebdfc03211b3b836fc0045942 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.04149135621677754, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0026386378608042917 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.07750948792703138, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004477188850582034 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.049656632315842844, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002856336743990567 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.008961527810790616, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0009890519095079862 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.017840955869101484, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0016844183124156535 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.010989532389756859, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.001029101618129444 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.03154250432068281, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0020252442164337293 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.05909979811818957, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0034347562251169 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.0375802335882974, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.002147921108027172 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.03303780727965199, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.002118654737857532 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.06214884490779202, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.003663125602489163 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.03944906528704634, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0022725008097993537 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.8321034229031437, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.08945851965278599 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_5.json b/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..d31168fee02f4af1c1d57ed76ced5f5b2037e5d8 --- /dev/null +++ b/perplexity/evaluation/generation/slim.perplexity_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.003117323860723007, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0008837148810752774 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.0026242515972514074, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0007336964722264186 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.0028022236632419895, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0007863889401033125 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.0005615954214733349, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.00024697557291310967 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.0004889607248097814, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.00023161252427707105 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.0005154358964062439, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.00023513093079516412 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.002278284775437062, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0006480855590254838 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.0019397313262420287, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0005430737577124271 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.002054859718325845, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0005764790804455003 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.0024716725936861457, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0007191187737619554 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.0020845273284634726, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0005920007456566033 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.002218076936186532, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0006333184346671743 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 3.1493567208404783e-39, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 1.4249121699400145e-34 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/rankeval/perplexity_0.json b/perplexity/evaluation/rankeval/perplexity_0.json new file mode 100644 index 0000000000000000000000000000000000000000..51ee7ad36cdab2ab858e2033bdfaed063e45d87d --- /dev/null +++ b/perplexity/evaluation/rankeval/perplexity_0.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.335, + "acc_stderr": 0.014933117490932573 + }, + "anli_r2": { + "acc": 0.332, + "acc_stderr": 0.014899597242811476 + }, + "anli_r3": { + "acc": 0.33666666666666667, + "acc_stderr": 0.01364760294240639 + }, + "cb": { + "acc": 0.4107142857142857, + "acc_stderr": 0.0663363415035954, + "f1": 0.1940928270042194 + }, + "copa": { + "acc": 0.79, + "acc_stderr": 0.040936018074033256 + }, + "hellaswag": { + "acc": 0.4989046006771559, + "acc_stderr": 0.004989769436956927, + "acc_norm": 0.6554471220872337, + "acc_norm_stderr": 0.004742510354777903 + }, + "rte": { + "acc": 0.5342960288808665, + "acc_stderr": 0.030025579819366426 + }, + "winogrande": { + "acc": 0.5895816890292028, + "acc_stderr": 0.013825107120035865 + }, + "storycloze_2016": { + "acc": 0.7349011223944415, + "acc_stderr": 0.01020698782076139 + }, + "boolq": { + "acc": 0.5923547400611621, + "acc_stderr": 0.008594580270731613 + }, + "arc_easy": { + "acc": 0.6031144781144782, + "acc_stderr": 0.010039236800583209, + "acc_norm": 0.5332491582491582, + "acc_norm_stderr": 0.010237073872130745 + }, + "arc_challenge": { + "acc": 0.28498293515358364, + "acc_stderr": 0.013191348179838793, + "acc_norm": 0.3037542662116041, + "acc_norm_stderr": 0.013438909184778757 + }, + "sciq": { + "acc": 0.862, + "acc_stderr": 0.010912152632504411, + "acc_norm": 0.782, + "acc_norm_stderr": 0.013063179040595296 + }, + "piqa": { + "acc": 0.7616974972796517, + "acc_stderr": 0.009940334245876207, + "acc_norm": 0.7693144722524483, + "acc_norm_stderr": 0.009828959550983103 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/rankeval/perplexity_0_lm-eval_global_step80108_2023-05-13-09-53-07_0shots_backup.json b/perplexity/evaluation/rankeval/perplexity_0_lm-eval_global_step80108_2023-05-13-09-53-07_0shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..51ee7ad36cdab2ab858e2033bdfaed063e45d87d --- /dev/null +++ b/perplexity/evaluation/rankeval/perplexity_0_lm-eval_global_step80108_2023-05-13-09-53-07_0shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.335, + "acc_stderr": 0.014933117490932573 + }, + "anli_r2": { + "acc": 0.332, + "acc_stderr": 0.014899597242811476 + }, + "anli_r3": { + "acc": 0.33666666666666667, + "acc_stderr": 0.01364760294240639 + }, + "cb": { + "acc": 0.4107142857142857, + "acc_stderr": 0.0663363415035954, + "f1": 0.1940928270042194 + }, + "copa": { + "acc": 0.79, + "acc_stderr": 0.040936018074033256 + }, + "hellaswag": { + "acc": 0.4989046006771559, + "acc_stderr": 0.004989769436956927, + "acc_norm": 0.6554471220872337, + "acc_norm_stderr": 0.004742510354777903 + }, + "rte": { + "acc": 0.5342960288808665, + "acc_stderr": 0.030025579819366426 + }, + "winogrande": { + "acc": 0.5895816890292028, + "acc_stderr": 0.013825107120035865 + }, + "storycloze_2016": { + "acc": 0.7349011223944415, + "acc_stderr": 0.01020698782076139 + }, + "boolq": { + "acc": 0.5923547400611621, + "acc_stderr": 0.008594580270731613 + }, + "arc_easy": { + "acc": 0.6031144781144782, + "acc_stderr": 0.010039236800583209, + "acc_norm": 0.5332491582491582, + "acc_norm_stderr": 0.010237073872130745 + }, + "arc_challenge": { + "acc": 0.28498293515358364, + "acc_stderr": 0.013191348179838793, + "acc_norm": 0.3037542662116041, + "acc_norm_stderr": 0.013438909184778757 + }, + "sciq": { + "acc": 0.862, + "acc_stderr": 0.010912152632504411, + "acc_norm": 0.782, + "acc_norm_stderr": 0.013063179040595296 + }, + "piqa": { + "acc": 0.7616974972796517, + "acc_stderr": 0.009940334245876207, + "acc_norm": 0.7693144722524483, + "acc_norm_stderr": 0.009828959550983103 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/rankeval/perplexity_1.json b/perplexity/evaluation/rankeval/perplexity_1.json new file mode 100644 index 0000000000000000000000000000000000000000..0d25f2f091c078cb54be8ffcac53ce1a1428e0cd --- /dev/null +++ b/perplexity/evaluation/rankeval/perplexity_1.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.334, + "acc_stderr": 0.014922019523732958 + }, + "anli_r2": { + "acc": 0.322, + "acc_stderr": 0.014782913600996673 + }, + "anli_r3": { + "acc": 0.3425, + "acc_stderr": 0.013704669762934727 + }, + "cb": { + "acc": 0.4642857142857143, + "acc_stderr": 0.06724777654937658, + "f1": 0.34145702306079667 + }, + "copa": { + "acc": 0.81, + "acc_stderr": 0.03942772444036623 + }, + "hellaswag": { + "acc": 0.49591714797849035, + "acc_stderr": 0.004989615052547472, + "acc_norm": 0.6609241187014538, + "acc_norm_stderr": 0.004724281487819379 + }, + "rte": { + "acc": 0.5595667870036101, + "acc_stderr": 0.029882123363118716 + }, + "winogrande": { + "acc": 0.5935280189423836, + "acc_stderr": 0.013804448697753375 + }, + "storycloze_2016": { + "acc": 0.7279529663281668, + "acc_stderr": 0.010290888060871242 + }, + "boolq": { + "acc": 0.6085626911314985, + "acc_stderr": 0.008536430524403954 + }, + "arc_easy": { + "acc": 0.6266835016835017, + "acc_stderr": 0.009925009142802914, + "acc_norm": 0.5782828282828283, + "acc_norm_stderr": 0.010133255284012316 + }, + "arc_challenge": { + "acc": 0.3037542662116041, + "acc_stderr": 0.013438909184778755, + "acc_norm": 0.32849829351535836, + "acc_norm_stderr": 0.013724978465537363 + }, + "sciq": { + "acc": 0.906, + "acc_stderr": 0.009233052000787733, + "acc_norm": 0.887, + "acc_norm_stderr": 0.010016552866696851 + }, + "piqa": { + "acc": 0.7562568008705114, + "acc_stderr": 0.010017199471500617, + "acc_norm": 0.7627856365614799, + "acc_norm_stderr": 0.009924694933586371 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/rankeval/perplexity_1_lm-eval_global_step80108_2023-05-13-09-53-07_1shots_backup.json b/perplexity/evaluation/rankeval/perplexity_1_lm-eval_global_step80108_2023-05-13-09-53-07_1shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..0d25f2f091c078cb54be8ffcac53ce1a1428e0cd --- /dev/null +++ b/perplexity/evaluation/rankeval/perplexity_1_lm-eval_global_step80108_2023-05-13-09-53-07_1shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.334, + "acc_stderr": 0.014922019523732958 + }, + "anli_r2": { + "acc": 0.322, + "acc_stderr": 0.014782913600996673 + }, + "anli_r3": { + "acc": 0.3425, + "acc_stderr": 0.013704669762934727 + }, + "cb": { + "acc": 0.4642857142857143, + "acc_stderr": 0.06724777654937658, + "f1": 0.34145702306079667 + }, + "copa": { + "acc": 0.81, + "acc_stderr": 0.03942772444036623 + }, + "hellaswag": { + "acc": 0.49591714797849035, + "acc_stderr": 0.004989615052547472, + "acc_norm": 0.6609241187014538, + "acc_norm_stderr": 0.004724281487819379 + }, + "rte": { + "acc": 0.5595667870036101, + "acc_stderr": 0.029882123363118716 + }, + "winogrande": { + "acc": 0.5935280189423836, + "acc_stderr": 0.013804448697753375 + }, + "storycloze_2016": { + "acc": 0.7279529663281668, + "acc_stderr": 0.010290888060871242 + }, + "boolq": { + "acc": 0.6085626911314985, + "acc_stderr": 0.008536430524403954 + }, + "arc_easy": { + "acc": 0.6266835016835017, + "acc_stderr": 0.009925009142802914, + "acc_norm": 0.5782828282828283, + "acc_norm_stderr": 0.010133255284012316 + }, + "arc_challenge": { + "acc": 0.3037542662116041, + "acc_stderr": 0.013438909184778755, + "acc_norm": 0.32849829351535836, + "acc_norm_stderr": 0.013724978465537363 + }, + "sciq": { + "acc": 0.906, + "acc_stderr": 0.009233052000787733, + "acc_norm": 0.887, + "acc_norm_stderr": 0.010016552866696851 + }, + "piqa": { + "acc": 0.7562568008705114, + "acc_stderr": 0.010017199471500617, + "acc_norm": 0.7627856365614799, + "acc_norm_stderr": 0.009924694933586371 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/rankeval/perplexity_2.json b/perplexity/evaluation/rankeval/perplexity_2.json new file mode 100644 index 0000000000000000000000000000000000000000..b8e9fc97f2b453725f53865cb997ee2656483126 --- /dev/null +++ b/perplexity/evaluation/rankeval/perplexity_2.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.33, + "acc_stderr": 0.014876872027456732 + }, + "anli_r2": { + "acc": 0.341, + "acc_stderr": 0.014998131348402702 + }, + "anli_r3": { + "acc": 0.32166666666666666, + "acc_stderr": 0.013490095282989526 + }, + "cb": { + "acc": 0.3392857142857143, + "acc_stderr": 0.06384226561930825, + "f1": 0.2736908716975162 + }, + "copa": { + "acc": 0.78, + "acc_stderr": 0.04163331998932262 + }, + "hellaswag": { + "acc": 0.4962158932483569, + "acc_stderr": 0.004989638507409918, + "acc_norm": 0.6642103166699861, + "acc_norm_stderr": 0.004713006072807722 + }, + "rte": { + "acc": 0.5415162454873647, + "acc_stderr": 0.029992535385373314 + }, + "winogrande": { + "acc": 0.5935280189423836, + "acc_stderr": 0.013804448697753375 + }, + "storycloze_2016": { + "acc": 0.7338321753073223, + "acc_stderr": 0.010220104800551206 + }, + "boolq": { + "acc": 0.6244648318042814, + "acc_stderr": 0.008469774334938068 + }, + "arc_easy": { + "acc": 0.6296296296296297, + "acc_stderr": 0.009908978578665757, + "acc_norm": 0.6123737373737373, + "acc_norm_stderr": 0.00999730791444761 + }, + "arc_challenge": { + "acc": 0.30887372013651876, + "acc_stderr": 0.013501770929344003, + "acc_norm": 0.3302047781569966, + "acc_norm_stderr": 0.013743085603760427 + }, + "sciq": { + "acc": 0.916, + "acc_stderr": 0.008776162089491122, + "acc_norm": 0.898, + "acc_norm_stderr": 0.009575368801653902 + }, + "piqa": { + "acc": 0.7573449401523396, + "acc_stderr": 0.010002002569708698, + "acc_norm": 0.7714907508161044, + "acc_norm_stderr": 0.009796313511829512 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/rankeval/perplexity_2_lm-eval_global_step80108_2023-05-13-09-53-07_2shots_backup.json b/perplexity/evaluation/rankeval/perplexity_2_lm-eval_global_step80108_2023-05-13-09-53-07_2shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..b8e9fc97f2b453725f53865cb997ee2656483126 --- /dev/null +++ b/perplexity/evaluation/rankeval/perplexity_2_lm-eval_global_step80108_2023-05-13-09-53-07_2shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.33, + "acc_stderr": 0.014876872027456732 + }, + "anli_r2": { + "acc": 0.341, + "acc_stderr": 0.014998131348402702 + }, + "anli_r3": { + "acc": 0.32166666666666666, + "acc_stderr": 0.013490095282989526 + }, + "cb": { + "acc": 0.3392857142857143, + "acc_stderr": 0.06384226561930825, + "f1": 0.2736908716975162 + }, + "copa": { + "acc": 0.78, + "acc_stderr": 0.04163331998932262 + }, + "hellaswag": { + "acc": 0.4962158932483569, + "acc_stderr": 0.004989638507409918, + "acc_norm": 0.6642103166699861, + "acc_norm_stderr": 0.004713006072807722 + }, + "rte": { + "acc": 0.5415162454873647, + "acc_stderr": 0.029992535385373314 + }, + "winogrande": { + "acc": 0.5935280189423836, + "acc_stderr": 0.013804448697753375 + }, + "storycloze_2016": { + "acc": 0.7338321753073223, + "acc_stderr": 0.010220104800551206 + }, + "boolq": { + "acc": 0.6244648318042814, + "acc_stderr": 0.008469774334938068 + }, + "arc_easy": { + "acc": 0.6296296296296297, + "acc_stderr": 0.009908978578665757, + "acc_norm": 0.6123737373737373, + "acc_norm_stderr": 0.00999730791444761 + }, + "arc_challenge": { + "acc": 0.30887372013651876, + "acc_stderr": 0.013501770929344003, + "acc_norm": 0.3302047781569966, + "acc_norm_stderr": 0.013743085603760427 + }, + "sciq": { + "acc": 0.916, + "acc_stderr": 0.008776162089491122, + "acc_norm": 0.898, + "acc_norm_stderr": 0.009575368801653902 + }, + "piqa": { + "acc": 0.7573449401523396, + "acc_stderr": 0.010002002569708698, + "acc_norm": 0.7714907508161044, + "acc_norm_stderr": 0.009796313511829512 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/rankeval/perplexity_3.json b/perplexity/evaluation/rankeval/perplexity_3.json new file mode 100644 index 0000000000000000000000000000000000000000..cfc0d52a414dfe31ff60c86918150d9818b47646 --- /dev/null +++ b/perplexity/evaluation/rankeval/perplexity_3.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.313, + "acc_stderr": 0.014671272822977888 + }, + "anli_r2": { + "acc": 0.356, + "acc_stderr": 0.015149042659306628 + }, + "anli_r3": { + "acc": 0.335, + "acc_stderr": 0.013630871843821472 + }, + "cb": { + "acc": 0.42857142857142855, + "acc_stderr": 0.06672848092813058, + "f1": 0.4014346362172449 + }, + "copa": { + "acc": 0.85, + "acc_stderr": 0.03588702812826371 + }, + "hellaswag": { + "acc": 0.4993029277036447, + "acc_stderr": 0.00498977656227611, + "acc_norm": 0.6642103166699861, + "acc_norm_stderr": 0.0047130060728077195 + }, + "rte": { + "acc": 0.5523465703971119, + "acc_stderr": 0.02993107036293953 + }, + "winogrande": { + "acc": 0.6006314127861089, + "acc_stderr": 0.013764933546717612 + }, + "storycloze_2016": { + "acc": 0.738107963655799, + "acc_stderr": 0.010167174759499612 + }, + "boolq": { + "acc": 0.6311926605504588, + "acc_stderr": 0.00843865607975907 + }, + "arc_easy": { + "acc": 0.6384680134680135, + "acc_stderr": 0.00985850654316206, + "acc_norm": 0.6195286195286195, + "acc_norm_stderr": 0.00996230599205857 + }, + "arc_challenge": { + "acc": 0.31569965870307165, + "acc_stderr": 0.013582571095815291, + "acc_norm": 0.3387372013651877, + "acc_norm_stderr": 0.013830568927974332 + }, + "sciq": { + "acc": 0.923, + "acc_stderr": 0.008434580140240662, + "acc_norm": 0.913, + "acc_norm_stderr": 0.00891686663074589 + }, + "piqa": { + "acc": 0.7616974972796517, + "acc_stderr": 0.009940334245876207, + "acc_norm": 0.7720348204570185, + "acc_norm_stderr": 0.009788093832324908 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/rankeval/perplexity_3_lm-eval_global_step80108_2023-05-13-09-53-07_3shots_backup.json b/perplexity/evaluation/rankeval/perplexity_3_lm-eval_global_step80108_2023-05-13-09-53-07_3shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..cfc0d52a414dfe31ff60c86918150d9818b47646 --- /dev/null +++ b/perplexity/evaluation/rankeval/perplexity_3_lm-eval_global_step80108_2023-05-13-09-53-07_3shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.313, + "acc_stderr": 0.014671272822977888 + }, + "anli_r2": { + "acc": 0.356, + "acc_stderr": 0.015149042659306628 + }, + "anli_r3": { + "acc": 0.335, + "acc_stderr": 0.013630871843821472 + }, + "cb": { + "acc": 0.42857142857142855, + "acc_stderr": 0.06672848092813058, + "f1": 0.4014346362172449 + }, + "copa": { + "acc": 0.85, + "acc_stderr": 0.03588702812826371 + }, + "hellaswag": { + "acc": 0.4993029277036447, + "acc_stderr": 0.00498977656227611, + "acc_norm": 0.6642103166699861, + "acc_norm_stderr": 0.0047130060728077195 + }, + "rte": { + "acc": 0.5523465703971119, + "acc_stderr": 0.02993107036293953 + }, + "winogrande": { + "acc": 0.6006314127861089, + "acc_stderr": 0.013764933546717612 + }, + "storycloze_2016": { + "acc": 0.738107963655799, + "acc_stderr": 0.010167174759499612 + }, + "boolq": { + "acc": 0.6311926605504588, + "acc_stderr": 0.00843865607975907 + }, + "arc_easy": { + "acc": 0.6384680134680135, + "acc_stderr": 0.00985850654316206, + "acc_norm": 0.6195286195286195, + "acc_norm_stderr": 0.00996230599205857 + }, + "arc_challenge": { + "acc": 0.31569965870307165, + "acc_stderr": 0.013582571095815291, + "acc_norm": 0.3387372013651877, + "acc_norm_stderr": 0.013830568927974332 + }, + "sciq": { + "acc": 0.923, + "acc_stderr": 0.008434580140240662, + "acc_norm": 0.913, + "acc_norm_stderr": 0.00891686663074589 + }, + "piqa": { + "acc": 0.7616974972796517, + "acc_stderr": 0.009940334245876207, + "acc_norm": 0.7720348204570185, + "acc_norm_stderr": 0.009788093832324908 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/rankeval/perplexity_4.json b/perplexity/evaluation/rankeval/perplexity_4.json new file mode 100644 index 0000000000000000000000000000000000000000..8fb072a6f48b03d0a5fb0fa01fa67ff1e635cbf5 --- /dev/null +++ b/perplexity/evaluation/rankeval/perplexity_4.json @@ -0,0 +1,59 @@ +{ + "results": { + "anli_r1": { + "acc": 0.341, + "acc_stderr": 0.014998131348402709 + }, + "anli_r2": { + "acc": 0.358, + "acc_stderr": 0.01516792886540756 + }, + "anli_r3": { + "acc": 0.3516666666666667, + "acc_stderr": 0.013789711695404794 + }, + "cb": { + "acc": 0.4642857142857143, + "acc_stderr": 0.06724777654937658, + "f1": 0.41697135221649567 + }, + "copa": { + "acc": 0.8, + "acc_stderr": 0.040201512610368445 + }, + "hellaswag": { + "acc": 0.4998008364867556, + "acc_stderr": 0.0049897810155954715, + "acc_norm": 0.6693885680143398, + "acc_norm_stderr": 0.004694718918225764 + }, + "rte": { + "acc": 0.5523465703971119, + "acc_stderr": 0.02993107036293953 + }, + "winogrande": { + "acc": 0.5887924230465666, + "acc_stderr": 0.013829128358676862 + }, + "storycloze_2016": { + "acc": 0.7471940138963121, + "acc_stderr": 0.01005054390987858 + }, + "boolq": { + "acc": 0.6425076452599389, + "acc_stderr": 0.008382336069484898 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/rankeval/perplexity_4_lm-eval_global_step80108_2023-05-13-09-53-07_4shots_backup.json b/perplexity/evaluation/rankeval/perplexity_4_lm-eval_global_step80108_2023-05-13-09-53-07_4shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..8fb072a6f48b03d0a5fb0fa01fa67ff1e635cbf5 --- /dev/null +++ b/perplexity/evaluation/rankeval/perplexity_4_lm-eval_global_step80108_2023-05-13-09-53-07_4shots_backup.json @@ -0,0 +1,59 @@ +{ + "results": { + "anli_r1": { + "acc": 0.341, + "acc_stderr": 0.014998131348402709 + }, + "anli_r2": { + "acc": 0.358, + "acc_stderr": 0.01516792886540756 + }, + "anli_r3": { + "acc": 0.3516666666666667, + "acc_stderr": 0.013789711695404794 + }, + "cb": { + "acc": 0.4642857142857143, + "acc_stderr": 0.06724777654937658, + "f1": 0.41697135221649567 + }, + "copa": { + "acc": 0.8, + "acc_stderr": 0.040201512610368445 + }, + "hellaswag": { + "acc": 0.4998008364867556, + "acc_stderr": 0.0049897810155954715, + "acc_norm": 0.6693885680143398, + "acc_norm_stderr": 0.004694718918225764 + }, + "rte": { + "acc": 0.5523465703971119, + "acc_stderr": 0.02993107036293953 + }, + "winogrande": { + "acc": 0.5887924230465666, + "acc_stderr": 0.013829128358676862 + }, + "storycloze_2016": { + "acc": 0.7471940138963121, + "acc_stderr": 0.01005054390987858 + }, + "boolq": { + "acc": 0.6425076452599389, + "acc_stderr": 0.008382336069484898 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/rankeval/perplexity_5.json b/perplexity/evaluation/rankeval/perplexity_5.json new file mode 100644 index 0000000000000000000000000000000000000000..2bda25e2dce8eb2d4ad66d845ec81e8551e44dd8 --- /dev/null +++ b/perplexity/evaluation/rankeval/perplexity_5.json @@ -0,0 +1,54 @@ +{ + "results": { + "anli_r1": { + "acc": 0.315, + "acc_stderr": 0.014696631960792487 + }, + "anli_r2": { + "acc": 0.333, + "acc_stderr": 0.014910846164229863 + }, + "anli_r3": { + "acc": 0.32666666666666666, + "acc_stderr": 0.013544340907003667 + }, + "cb": { + "acc": 0.39285714285714285, + "acc_stderr": 0.0658538889806635, + "f1": 0.3398268398268398 + }, + "copa": { + "acc": 0.8, + "acc_stderr": 0.040201512610368445 + }, + "hellaswag": { + "acc": 0.4982075283808006, + "acc_stderr": 0.004989749347461089, + "acc_norm": 0.6702848038239394, + "acc_norm_stderr": 0.004691488813032134 + }, + "rte": { + "acc": 0.5703971119133574, + "acc_stderr": 0.02979666882912467 + }, + "winogrande": { + "acc": 0.5935280189423836, + "acc_stderr": 0.013804448697753375 + }, + "storycloze_2016": { + "acc": 0.7477284874398717, + "acc_stderr": 0.010043504206387307 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0 + } +} \ No newline at end of file diff --git a/perplexity/evaluation/rankeval/perplexity_5_lm-eval_global_step80108_2023-05-13-09-53-07_5shots_backup.json b/perplexity/evaluation/rankeval/perplexity_5_lm-eval_global_step80108_2023-05-13-09-53-07_5shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..2bda25e2dce8eb2d4ad66d845ec81e8551e44dd8 --- /dev/null +++ b/perplexity/evaluation/rankeval/perplexity_5_lm-eval_global_step80108_2023-05-13-09-53-07_5shots_backup.json @@ -0,0 +1,54 @@ +{ + "results": { + "anli_r1": { + "acc": 0.315, + "acc_stderr": 0.014696631960792487 + }, + "anli_r2": { + "acc": 0.333, + "acc_stderr": 0.014910846164229863 + }, + "anli_r3": { + "acc": 0.32666666666666666, + "acc_stderr": 0.013544340907003667 + }, + "cb": { + "acc": 0.39285714285714285, + "acc_stderr": 0.0658538889806635, + "f1": 0.3398268398268398 + }, + "copa": { + "acc": 0.8, + "acc_stderr": 0.040201512610368445 + }, + "hellaswag": { + "acc": 0.4982075283808006, + "acc_stderr": 0.004989749347461089, + "acc_norm": 0.6702848038239394, + "acc_norm_stderr": 0.004691488813032134 + }, + "rte": { + "acc": 0.5703971119133574, + "acc_stderr": 0.02979666882912467 + }, + "winogrande": { + "acc": 0.5935280189423836, + "acc_stderr": 0.013804448697753375 + }, + "storycloze_2016": { + "acc": 0.7477284874398717, + "acc_stderr": 0.010043504206387307 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0 + } +} \ No newline at end of file diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..892b1695c8c616ce1ee2e03ada41e51db1d100a2 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ef7a714d725648d82fe2348d3dc76c0e8444be1f276dbab2362ad405088c7a1 +size 199058647 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e12d8cf3048cc79857b63944559a63fcf61ef90b --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:661787828ec6046fc21863a6842b12d6bc411f91f1e2fa0ad74da6a29dcf938e +size 199058647 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a39c53ae92ded08da53400b37dd50f388b8d0929 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce7db285d0d90090596283e13ca6747e5d821ffb6b2e5a35ee44ef2f8748d906 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df0beeac0b0918b060861e7b9fc8d47afb48b0f2 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abaaba21a14911a91b7186f96b9ff6d13cf348ca5558e9b94ebcccc160e2f99f +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c26ce6472daee093cd6e89ffdfc1a3bd5f3ed428 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b0cd233fcd43ad44bc5e4a2f482ed56c6617b1e28e75e077847f27e5edf4800 +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7446b6772f68e9adde9c0c557dd4c2ea372e9014 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7b81877dde9af665118e67c7430fb429a9e94e6cd24b4be9fa040ac14a76c9b +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ca1e9761ede5f3e4e35da4951f1cd121d0ad7a3 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f85de669d518f2fa5b11a896277da8f49380726a4f6cc9ec34b306dc46e76b29 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e856f115a5266b3dab85a63ef301acf305c6c0a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e688fae789961665dcbbdb410aad70da46705058a64e04969599d5a466c66e0e +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f5838bd2d9b10542872b6cb44942dedc9b96707 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ea864f5aae90cd7b5264ba5df49f05d2f51042b1a74ac7864c3510a5b1c8ce0 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab941604d546191138fc0cbb2a7f116dc800f417 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c856dc5ed3a01d074001eca36cf344d23f4d7308aa7e39f19a2529770b80528b +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5837c204d6b40e4fe4259213dd44aaf1bdebed5f --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb3982922eb9ccd23f319291e47248f892cbe67958f11301a282d666217adc5 +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9136145197be873a11459be3ffe50f1a19ab0f13 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dfef78f4edef4d1460e257a5b3a43001b7430d587ef637702e62d63a8d6ce0e +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ad02590e6a2fc29d1d5e0639586194f269b8f1d --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d591323ca075177308e799498b33816dee9deeb610da3c6e06947090abebd091 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b9f0c68af579e3c73de40190f4befdf468f87a0 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7688e2b6a3439f62e8252c4006e06cb3ea391037ba92a283bb9fb3ca501ca694 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7ff644250c027576f5707eb7bba11302eeea13d --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74a0888699ebada0df81776e027c27c68a9996b2f6f178faa1bf29877759c7c4 +size 199058669 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9e979d4cbf299271edfd6ac1bf239b74ddb0e55 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d311a0b2fd6a94e0201ad203426b138e4ea9cb7f31ed9a66c922e76ff67616f9 +size 199058669 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7499fa1a5e09b83c3c7bde45281e3a0dcb56f845 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f790863b8f3c34741b6a72d458b1dbef5a582c99091adccdbbc95db790fb046f +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de2cd63df5072a384b898dc5829d9b7cc08dedbb --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f696991cd2b09226d4ade1eff049c4d53e03fa158f094beb0fd6c2f78f6cc245 +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94b4c59e00fe04617851a90d178b85ed7e115155 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6b8a044ce54bd789258f17fb787df7631f112630f9e522efddddf58fabaa81f +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3655f9a9d97a7aa696c42ff5f8ff010047d79a35 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5a343006d17dbf1d334c2611961e3690baf0786799d77f38e3d3d55c3e60878 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5df607ae0984be52db4e09b6ab69167cec83e82 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22e2ad36192c516dc37f38e90a70e21a9ac5310a66be218bb8f51b4474764c1d +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6688551d102d37e4906a6bc3739f9f1a796e1567 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38bd6488e3dad119a00ea4dd01a4ae0bcefa1d119abf12a12fe9589a1d31c150 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7df82e1060144b14bd3276bbdab02f9dd04f84bc --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b661264378078d2a1d1f500210abb3c2d054d24af6665910d312dba20441ffcf +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bbc26d8ae401774c9ed65f16252ee6b56459386 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:023d58bf80c1627d9b0f4a8b2872373b23f8108e7679708c10ce57d163ee9896 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9c004ea14918699691badafce9ea1042f8d8bbe --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7c53d3b78555f9c694e556371830c870b270d97e4affea188e22a229f6c9864 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..84953fb775980faf1deb1ee046a6cce345e51f68 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca5510f754ca36d91a0f14ba698603b9a06032dc65a42a1da09bcf5f082bc232 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c452f722e95972be3c7afedb62c245449a3745a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fba3cee1d142687eb67dcffcbf55c624c7aaed43b6ab3563a6a15ff7098899b3 +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7426d2a7949c1c879ae99281fa35472a2765ee79 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcc16a683532c49ab314d03a0d0ab963dfb9554b68928c5c9e42f59bf9e94083 +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03dc7117ef8eb553068db0f458c331a7a04da9d0 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4765d787e5073acc57c26dcdbf4b6a97db71ce6dda5e067fec9e2fc05826a472 +size 199058605 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1a154d15125ee9a35aaa59f496522b10bffc27f --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a8abcd2a8d341051d1f9308847a4c263c6d347dfc9105956617e1932aaf1615 +size 199058605 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ddfe9e0d5b2b563c281bdead5599538714835bd --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b09fbf44c6fca46f0b5c4968881217ae7a5ae6dc0e187f58042f019377c86396 +size 199058669 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6058c8e62d515b494a7329108e1349f2cdf47be9 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea8ba757df078be5727ac35b0be8f5f7f998f35160818531b5946a702703dd4b +size 199058669 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5775f6a82c15a8767982e84193dcb820a5c24b6 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:060d9d0a81a9897b23f1392aa9d610ea195352cd7c94685d8f3408fdf5f32ed8 +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3b73465b238cf53a9c708c49a882d41b13f39fd --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb721c6f15207de0bb85b634edb0f1e46e97b6af0e7eeaf334ace7d32cb3bfec +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..730556476101e2ddc5d4c4ef654eeb2b4243f136 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67466e4f383cdcf62d2e3abc5d7a9209bdef252b99842a1544c9bdc1da149bfd +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3782f674f31e2e230d1498f74ebec2f831d2ec1 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:200f3e6732a3c7e2165d51dd0ae2d61c50eb520e5ae58fb33b2ce361c9d8aaf8 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf49a4fa429b58fb00148b3a7f1c754925249f74 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88fb811316597136a74b19d40926711921abb18b324b172dd63970dcbb271847 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b221bc27375aeea561e9287a1d4a94f48ea5d2f2 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e649fb798f6358485af7e9f68f5fed1edfcca4ae6eb4ac02a37300e1f8fc181 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e031d6431a13ee4701a30e2f5221e2cfc121149d --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a440ea1a0308708afcfd1e8db8d509470885884473986ee834c94643ad42a0d0 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed024071d78d5d23000e97094438c73669bce3ac --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d1bcc9d4702e88e74ec2b3396144f957e65405ce6ca6b27b1c062dde3ad4ede +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5fe0c659a89168f4a1262677028a9ae405e4a61 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa8f8021397b75b605376b32d53bf7ff3f2590d25d92496bba44688cc88d86cd +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ab63acf58acd323131450e8a1c706fa6e51aa3f --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:898266137f41c68d06f11269444915727d62dea95a514929eb9be20e40070bca +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9e8d395e69b82712bc3fd230d5334b20bfb79f2 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1421556ec9e7cc2a2b939125ea9243c06786c3e23c6f1ea446d7156cf2ae7bb5 +size 199058669 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..96d49ca986bbea717e666dad4d5a550a00caccff --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56ac7043ef0b205cf938e09b2983dbcbf393e5649aae8439e88063fb1f4f94e9 +size 199058669 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61f871a0ec253a198185b97341920bd6350c1c58 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10740277d29d028605b2fd44f27d40de9f8dfdddc500bdaa4d0be9f9905aeb22 +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..51a9c4293d38c0abdc7bac137b24a49f44de69c3 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ea4ea88665ada918dc3e52be0305089d7634e0db7d18745a0ad7dda0d701cc9 +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83efaa2f681acc23dbb3e69d5a794540d8075f2a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e6833d9010edaf447abd41dcca3abbbd8fb7a51610d630f2864246ad144f65a +size 199058669 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc15ebaee432465dd96f98f61bfb48286ec4583a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c7f0f9a6f02be005b70ea0996eefdc16eaec9fe419d8fb74e3d31ffcaa5045d +size 199058669 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a67ce04ea3d5a7c5f74d736e8243baf96c20674b --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d239e78d568a969601e4c81cf8b0b8bd20f174269340cda6b30030d048ae5f38 +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6a58a5fbce302a6c8f915f78c54cb17f16db8ad --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f121b97833f3e65e557df3092122b7b132b2183216134e36db0068a291d61945 +size 199058797 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..366175d7dd264d0e37675bf8ebf3ee27e337db10 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc612f8528cd53478111eca0e27bc675b298eff8f4d5a3652a9f928fc58ff106 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5000799073c998ed5e9842198738eeccbcdd028e --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6927bf931af77970e09d4bb443909db9b621f1ce6fb6d8294c99404e94ed513 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f3dd8117ecc9f946e3bba1945129989a2efbb66 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:475bed55713bef73f7e484e7755909f3ad2c864f8156ace36fa1100b244133f1 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e6c53a2a298a76499654481a9277a1955409987 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44e022c455377fbc143f42f1505af7675e2fd0daf9839f32876e330d3e78f986 +size 199058733 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62da82f111951249894feed487753aaca56da2df --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66d6c431f75c4e9baab3d66a74b33d42808797db0806d9f3ed265bf827ea55d6 +size 199058669 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca950d38f8e1665361748d6def4862d49d47c949 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb3ce6e077ff0a421c052fa2a94c064fdcad0480148e1b9ba73eaed9902e9bda +size 199058669 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f972ab8428877463f7092d596cd16e02dcfe8a9 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4698c5369f1450da20053a600e34f1b6ab50ae8b04b35dff5b17d6334403741 +size 199058925 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9fed74f503ccf6969480f223501a689d17eb8745 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2697320a83e583790b3de37bca4f9584f869c43c137b6961aa5b874f146f349c +size 199058925 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ac18123eb471b441eef918d6b582399c61bb867 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26bba3305c83b437c8fbbbeb1b7e896ac35b1e37623ab42f3e325f502a762494 +size 199058605 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ac021142aa039c8f898190b195b36933cb34a37 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c3b8fbe2c2bb14a487da2f4b9f2857a38a0be5f15737d595c14f5589c785715 +size 199058605 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cad3599317777dd24539e9c23e3a18e2be2a8b15 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29dfed2f15039e77598ed888d0f7f064653c26b012c496929779e53bbfe2bb81 +size 199058605 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..78b97e87ee83d0bc6b6698bc7e3e0d6b7b6a492a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b09515960af7e5d0885270ef818fdbbf982c26576de915c2b3d2c5e7ba3b7348 +size 199058605 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e941eeac14e06c3ea94deb3be0bd5e71291ca78d --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeefdb6b7a6c3b15338850777161abd53308d3e1a1b5562ea94d2103e2460dcb +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f92185860dd4e656126e4b14cac0d92668cdde37 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5070a53ff9e8e68209f3afd22f0feaabdb262f7ccad84ed3c1f2dd97774e4ee +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7f8990d983419e9ff8fef95188bc4eb2d057d32 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c7f250dec5d3e4f25eab07f647a82fe26706f2fef669b41d79dde52ccbc1226 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8b2865fdcb30ac8db0d439f367802c4a6d23674 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40359aadabd97e49b1204a63d2beabb5c6580f3a026ab83410f754749a821f19 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..497007c7d700b5b2edf72d90d486ed90ac8ad310 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9161da422b9b425a6af3489be2943034354657b7a7320f99b0ee1f8598631f2 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21cb732627d2c9022ced05208c7aa09e3269a4ab --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eea901e8023181320d6ec0180378032507bff61db3b8c0799050cc6f63f583f9 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9e67bdafcbdcad07b56385dc63a52e08fc3d7e0 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8644e3ebec61ab764a00f4804b8c63a4feccaf855f22c2fe9ad99f352661b987 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..78ac21052a862cd68639fd59104c997b43cb2de1 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2697e1c2fc7f13b3ca349a29fb380bd2cd75ea9a2b16edc7b8b3d1d4d6dcd860 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d6908f9f1061af1a1afd53114712bd793a369de --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a69d15963e3e314201aff6f9a5c7c57718ad9785595d1c6d3a505d45eaea04af +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf0342832df746da5191d8dbb03b32ee934c4b80 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a152841a6404fb0bdbac9119cced95c0e87a0585ee46f13a26976debb000ffc +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89bd40207e7b3377720cedda676985abde98f913 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd4617b2d00e0ca658f2fd2c849e85a7d5c1603a9c17f2326d1af41930172430 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0d8ff8ebbf9248a201f098e3a2543acce800d65 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43f47cad5a007d36437c9a544e119596cb2659e105703cb4efc51d5bfde8ec3a +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0dcd7b1810773d79ebca272349bd2a3162ddeab --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86467fc0ecb693905d8d2c1c68982972dddc6caf96ed40392aa1b56a123eb407 +size 199058978 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..929de69ab670c5dbe789a7d428abe774cdeb3dfe --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1498833cc665f163ac088aefc21f88e4b76899e800e410947f2975e7528d547b +size 199058978 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0bfaae5338598a9f5e60856984069f5fabc92ea --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:705d8940015bef6eb1ac17e81fc6ececa522823be91c3beb8290a53d415778ce +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..84b732215f04edcb90af22c7adb4840f4397347a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:492175f8a227f5d3ba08562fef269e8a2ada12cd9110c04c87848444621478b6 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..20277c55d18b4dd34e27d33f8ba4e7d7a04700d0 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b352f254aef18aa8322c19cf0bb1cc2e9e9de467b7e26e1832b1b94e3b6fff71 +size 199058647 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..48924952cd4a0edc8f1e0f4f96dd6295214058bd --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1571e34c7c5fd17f29ee617517a15f1b41af11b8cfc5c34f443b1aa596a82aa2 +size 199058647 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b032498a334f3e664be2eae3ad8a0c719f0f673c --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:855cd1115b092ff6c76589c16569060af054e6ecdc64e320700effa8c0d8c190 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a31d0fa55d17dbcf77ac2410568bb3e60f5d85e4 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:659e01ee82a283822141158e389f5b55857118b036e78bae9aa841c703c37271 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2e8887f58817fe1a223c24d23f7eaff2159a4e9 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc38311cf5ff79b0b29cf01e5a314f76f3399887b40cbf92b5772c484db1974f +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1270f8f64964d01bac3d56413f3ac75c2e949e77 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c53b722c29984991305bfb4355b672d0ebfffffbf3fb81ab712830f77008b2d7 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a981fead9445f704276128097ba0351a1d2ec02 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90477bcdf8654f28cefd7f0ee32f332b1b32708b5077070e6067f01f1eac9f70 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30712c5f179674a44ac9cde21e2a8b9f6d1811aa --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f47dd8a4b6f778dcfbb3f0f95ac207fa31b9115babb873f1336eef9f11195dc +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b01c8f68961d0e14a4fb3f9039172615ddbee07 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dfab98926f34a8e9610504f9cf0a3bf2e1842f79d9b55a32a0a8fef5f8c5b9e +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4433e205233b38333da151405d4ebbbd151cbb3d --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b4e262abfa1c6b98ae6b7f8426d2d8c376214ce5ebc6ca005a9515077862866 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2d3a8d87be82ec472bfa6ea95181b70f49eedd9 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea1f78539d5009a209ce464fe0776b180708644832932bd4891a1167fe3daa5d +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac20e4804da96bfb8c3ba00402489a4ad6be609d --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b12891f0a2cc8f25c0427b6b26dd17b06e7d245e38b2fc17d0196bea0c6bfe6 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de639902825c15dcb944d301318d9006cd04b6d9 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9c86f44770db36004fde4c47836ecb3f1ed915960679ca35d18c862308ca99b +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4f4b327fb8cc9835cb47c31423ae40df1bae8a6 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae95aeffdfc35735d0733982f31eca29bd936d95b577743afaea40e03098d5af +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb45314a6cccd900ce2197f6b3d675a1efbb9762 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7065909e2a0fdae3ac4f42bba579efae3a0ef98b721ce221b6f33dcb7fa1bc9a +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e67a6104306a6bc49bc843269563d4a8604b575d --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb3f284cd5ec2fc3258e13fb27bd5dc46f2d22a7d82db65dfea21d0ecfc2de8b +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9e25896028b8eb11dfb70ccb1c00b276eb269fa --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fd1e399b61ee88b15ba409bbaa3b1c4a2dbdf91a012faeb7d673d9513d2e65f +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..469c6b6e4dad5f9749ffd6b5b487b9c5204c2456 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caac9e9f258e9c6978d64af6e1369929d19bfea96247f454c82d4c2fc855c79a +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08dac36f79f35701fba8d8036701e4576b9b30b7 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20f6dabe12768c77e586fa75b222f59271e23fa056c4b580392197e80575ef60 +size 199058594 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d62df8a17e75216bf17769d74465c4097e57d5a4 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27dc9cc2b3aacfccaa3e6013d7faf501c0a49e3dfcbb3f4f3ad75793413a6efb +size 199058594 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c2fa8a67d6feaf007d89b4d8e2c41d7a4721b03 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:263fa19146c953428b43ef3fc9de1db7eb56ba68e316e53fa4f20f9db0ec63f3 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3402f517b44c704b5d8770f4d48bfc3d835b960 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a119f8db442f08985bbc4fa17297209797600d1c4fb0a93f388e2b55593f3d07 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ac4a821ac66391db0c78e40a4eaede73698e70a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5115e226504dd967a9c795899622c177d2c542ede978b12e053ddede7eb8ddf9 +size 199058711 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc19f1a71878cf3ed467210ca800435db54d5a8d --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0245622ae6a3eb3a7cf717a770d49834f55741146274132264417cc1834d5d5d +size 199058711 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa3d61ecec02415142d4bcac19e4fd3ba9a6c44a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5693bf8e72595d5da32b8d3709c4fabc593981f93d70c8db6171a7004376d50a +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..182832323d1d895e124fefebe0175c0fe0621047 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23ad48171194e1b0867c3d150fc30753af002ecdb5bf4e03bd880278cefabb55 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52a4fbd9dc5cccd847bfc7c45aecd08d8c79f1ca --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ca23b014b1e6d1b11810692eab8d0dd1a36ded88c3d87677387ddf8971e84b6 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d699fd154d6afbb6f843e1d6fb051dfd58fdfcc --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3899ef2ee07536b5a7874b7b3a1561f3e6550c8e1ffbf861fa00efa9688d42fc +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5c35eaa4510c93c274f3435d9a4e69969af4677 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f29ea3d20fb2363b9acbeb3c7df349f3eac01814dc8ebaae6d666c82f4746891 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9eb06c5919ac2d09d74d3714a1eb0728dd5e44b3 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:956da8c8221b7d27cb20f2e137e4208e7945956f65d69df2eb877e92f0060609 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e4834dd3a06b577468849ec23d648f5bb62cc62 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff255d72d32b05b8aa5d8c421ae0eef5a3557b7e4d197d2a11a9090b9e66754d +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3aa0948f9a6bcbf894b1fa5dcb4a73dfef03416e --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c98df190edaf473cba6327b1cced05b49637324da109bda447e8dcdd3f33ea6 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb52494dac8be4eb43f6259f319b9ae8b540da9b --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd11f545ecbe11752bc90b9eca986e67b50b169c27d4929fba51e78145e63fad +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8469df52f2f18f599fd33ff3d357a72710ed09ae --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc8203214003f1d7c91fe07d27700e376aac2d01df49f28514b304b9ac0f3e94 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..58312d9a1797281d2139f5d8b34fbaab8716ee92 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d25f96f670650b92ceae06bdc8eeb0fe0aaf4b2304ce391e2cbc1c80d1ee05f +size 199058594 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc608e35358950ab45a4dfa050b208142a51a557 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6157c8f149288b7978f0d84a44e730cbce4f64fbaceed5d1609e8a1169c4574 +size 199058594 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2e4de9f0669ee238ced8f2dfba6830f00c65344 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02a5e3d8a670099d38240e53d0646af7062a26dbc4ddc9798cd709fee77571f8 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86e3d93a62b138ed22a1443e78f9b3a7e80f3247 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4d678580a0dc8ca76f88d7e38272ee498009494b6d2a85260d35b6e63b0243b +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52e2d53f03132f0e8fb43fcb32a1e3a7fda8b0e7 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fb322fdda58d48f64d7a95559976c2bfdf4723261345561938b02fba8661b15 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5805c50c305bd7ae7a8015151be3177c67b9f545 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95748c8effc08791c519a0325981e92011df377f0999c926dd7862fde9b6546c +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14b4b12f66eaf6db218d6d480757da71b2d77b4a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d75d45ee327250ae8a8d727c357f684073f18871fa42bf3de8a005708900d004 +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4370cb8c34953664b27076dd1b95103c9bc1e16f --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37dfb98fa8b108d5c1c1701db9301901bce6a370afaf5c38f72337a68e0cb06f +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf84bc8efcf0e1f19473bf9a37752cd760ad2b11 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baceaf89d4a3278c93016ba657266c249d355adaf8fd48760369415d7c7f7adb +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..070bb1fd31d517e90e36b9e9465ba3c576300ae5 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97cc695727260eebcda82910bd86643f0d63c22598daa8305b19646d65e585d2 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad7207a6dd21b71704e46eb60819f4cbd15774d0 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d96e564141f9397b9e2e275ffce853633dd34e5d9774cc283748a9099da5c49 +size 199058647 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ea3398de930d99bed2100f2759e8e702a96eedc --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f48c15e274aa9857b04b13f36d680d14cf98200243ba57298e18a02bdeeb8382 +size 199058647 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c3977ed14f8c0247746f8e8f4b85d15547177a5 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6abcf0c200058c587c8ab6800c5e96599711377046ef12ff3d7ca49e599ebab0 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f04a3fc8a390298f45c28680f303a8fe8b9ecb19 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fea4c69b99477d644a47575bd57db5d4ab9036f78b24e7a39fad7cc28370f80 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22de2d0234665c83c215c278745e4e273fcc483a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f384997697abc8080df8dbc4e1f0720188c8a8e1b2ba0dd5a9308a97466bdf3 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..240793c35e834ee1217e3f08a78eee5ec3b1d64e --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:879c8326e9d32302dbc0bae3c72fd804a3020e7cf8625b83def6965ac78d7752 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e08158a1ddbf48688961329bb25adb4bfede85be --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1e885e174b556f11eaa893b167eb277b89987a0eaa3494c31e2664580ee9fe7 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0bbf76996e08d40b5d17b7027fb7488d24534650 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edbd4bcd4e36ee9ba17d2d0913d67b98b5b811f80ea8ba5304fb90daa3a7f937 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..339c8b396b54fb2977611ed2df970410eace2fb4 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:322f3fda375ed8139709024016094cb0c531f709d9c2069c948bda4467d37ffa +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8fc87fda56bdf0ba79d21f125117e751afc2543 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39029acde406e5b1851eb4ca5be973ede1d767b099e8fac9980c3d0dfc4af213 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30b28a2deac4be74b20176d7205c3db9c2adef98 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb1807fedd5aa044dffb6e8d70f2454f3a1622998166b03dcc99cab5cf8c6cfb +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31b6b2ed31b26c7b822dbeeece0fd7342b2cea45 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:346cfa13b3453c5ad67ee635681eace7788fed21ee9ab6ea91574dd034347e11 +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2eed3591c98f9eab9d6769cf8dc4d68b10c604d0 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e5720dc79c4e43fe18824d50e5321cbbbcf7a3b8cceed10f2bf3d21ab1f6e79 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6130c69c9cfa49799d705e8afb3448a84c317b79 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfe0c72d211448369b5c763f8db5cc9db15934e4640e769f77812e367c6e49da +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..71e773c8e25219166013fd1f60c1e126dc89ffca --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5967ebaabec41ffed863b852c08d9dac1ad3aea9a4da012a390e3719e2b12923 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eaf8f262ba49b7dae579ed82fc0a3d734e6edb63 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d55fc73d7351d59253e8f01d4328d9409e7c82e42fc9d605828d26fa564f1376 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35776c0e44050cac06fa56901081cf2a8906f049 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bf57dd7ddf3fbeca073ceca2180ca13065b996d109f19e49a2ce6690eec16b7 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..424f7f27a86e541cf94a6fd77d0775813bbebc0e --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a948293032045c8e11d29967f1ec76757ea91d1896161167015ddb47ce9ed048 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ce0ce803be90de7ccfe4ebf20a2fba515ded0f6 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd2d7773c1ccffcc8c81fb753a9ce73b4f4d097119713d1e7c9483a65d99b002 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c59dfb523467267ad97e482e966ac33a5d24685 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dffd790d1d1aea4e156ec51bc6843fa183f05f0b4fbaa7a524bc9e09aff6e7a7 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f705252610dc5ef36b20467407cb6d45c534b225 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6886f7d4a10d268bf8bc0a71d54adbc2fb82b54777f67e437f6a59437b87f771 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..300f324c312240b2685838bc3cb875a8202765cf --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76498830f3a48cbd643ae66d2a802aebb1b6bf82857310140ddbe9a06bf412f6 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..48a4022eb823f3c18bde7b9c9c128ea1c861bda7 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb329e0362ba8ed886bbb793fc5d9bb77fd75762004111d685f2176b58b55101 +size 199058775 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f83646827a6b3f3b58ff3e722632e85fa372f92 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7a0e3af385d20ac4cefbd9fd1cb3c6cebb5f98559bc3361c5ec776e119b05c9 +size 199058775 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0aa277612e27556fcdd6cbfebccb02f21238d91 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce6bd303cc344563469a1765fc6b88015617d282eb1581f3e6882302221d925a +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35b3efa8049d2f45afe66c6a8f9e713986c878eb --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c4208a8cfa4b63ed3c8c9f922622b67a69332fcaf7646ae6a6cf8c201dd3755 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..73fe6bb0abf9040a5bd3c92eded820f223273b9c --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51d6a64eac55e17e45742c2c1b88e09f5cc11f3353a39a367ac5bdfcd8e9c19a +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..750233fa7c8fc4b7680120e986ef309b58e32b5d --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dff74014b2b150ab03d5b7fffb1f6a3569b95ca2c2358eec58cf9a3785335c1 +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..02b1bec758ffc9d63deb90bf26add9592aa5a1c4 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edca03767b396284e6ced2b2f443549427d96a5a4f820cfb06365e48591b26bc +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1a81d1110e9b075d1a3a6f6aae470a4e7086780 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62b8865f308144f11a38e2788f23f9db1baf818251b6829054c8b8b3db38e2ca +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29ce9c97d846b6a27c801ff6968e6c391c823971 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee9f98a90e0b648f6b9ebb47d94ca725302c4ccdbc0a06085daab9dfe1715409 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd986517cf1cadd21f11396b23ad64790488c639 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ea57326b690f889155fbede39aa78adf78babba708a0a0201e79ce54f7edcd6 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50a217a0da69839cfc51ede6fe7f8831d9082643 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caeb510671917a06bccda18a8befa9f2936b425e33aaf596e7928352a632022b +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08bcaa8040dd967b2da29d5b749d09a0354b3f32 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf64e0b59a88ef5fdc14f06a3c581ff1c29f9d66b3159c3608713d79f4fbd7b0 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb94ace03cb1a5f9a50bb78ab85fd6ce94d5e81f --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dccaf071a3b011dde7bde233e4a15801d55229e63133d27a343853d9e0f43942 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ea93d4c04431b7e62722831741d1c7ef62b5800 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:229ab83a9d402a86adef28a0a575dd7491b64ffb4a3490ff77f7cadf9c67f416 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e6119ab73ce7d20584d711f096d6b180c498647 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bb6582e99d9e823d4bc29d626497e0a6254cbc35d8c16259a610aa67a2d50d2 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5d4dffffe025c41c22013b0300c6c832607ee5a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:526888e01cb78ae72fad95d2e18f6c5f637b84726745ed52ea681613388728fb +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd5c100d5d675012dd35b5dad6133a815df573d9 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b063b0ae0bb59bb2971ce3916d5c9492e400c74f23ff85a3f8680399b6abbb4 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0f0933616d171297076fb432670559bd1adc49f --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1686fefb4872b2fcffb5c26db521df1d8a87d5af4ebb2b30f9de7316bf8946b +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d3e131b0872482cf4042972f211046c8934dcd9 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0ea4cb3eb03d7e0a2632a37870157f72b3f1a7a2a7fca675369b8795b07d262 +size 199058914 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a734a1930a6c57b71677f175a3a833541bcc53e --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bb4043f73e1ff80783c4c0c7d6c1650ac5c1e53cd74ac9b4f699019f4377372 +size 199058914 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..312117608e734682a4c0f843153658005e454b88 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9da7d5c00d6873b36cb57b79f08da5de2dfd6ae9abc256e869043669c8b7fa8b +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c44d3c4a372314ef445be5a6c794f621493c6b1b --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d861773ede21ff0a28286c6dbca1183697441984cb9f2273c732fd26930e7355 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9cf25e61f8a1f51bf96979c7247838edf9add6a3 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faecc90a0a7c014a5b98e4957e0dfd142791fb42f7f0c40589bebd4c7b68987f +size 199058711 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c3380bfc8e8b3494b3f98059508d2eaf8c4a4ba --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f937f1286ee312a4cb47de29209a3f6e731d12408fb9a00a62a1b69c452876c +size 199058711 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3db631fc3976f4e775e738ae3cc69365579622cd --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6379a0c7cc0c2bbc27945add1dba5f148edf87705b08ff3e77a4e5b950f437a +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f40badfbb25c9898539097e8dc316d263b2e4be --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:530a5dde9319895d3347e5f592ba609d3b16a1d1d6696004f28bcf9a3c7cbaf5 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50f633d4149d293aeb1b72e758a4a5a773c021bb --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b606ae86b85aff729c3405385d21687b893041de8d7a82dc79e7c3f9e61c8bd +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4466cfaf19af2f90d5dc9088ea3c8aab2ee9f5fb --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f2319812245958a0168f02906eac9e63db093f4020812f82fb1cc78ea4082f4 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6b9ee3603497512b28bdaa9de70c131902431ef --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0b9b05b8ef2d3ddbfc5935c07db84a935b5e004ab6ab221eaecb2ac14f045c1 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a344d6ae151efcf5d7c1cc59e698e02a3d4bab39 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aedcb9db5f73c6c3de7f4d3bf63fca89a0e6238dbe069ec44c5c27816276387b +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd6fba5a89196a7377c7867edc37cd51ba421c15 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70de9bfe7d36cdc71390d697a0f8075d0a3faade8beb01e0c12a93e414efd03d +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f46a5f7a98af255d3003022c5bc93c2fb4bb1e49 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e0509d9d1eeba7a33b50630230686117111e493cb1edae8fc843337ffb505c0 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5dc7d0dfbbb6cd2e3d0ed98ac152f8b9c7c543e7 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a61354443c3a7814934ffa74df7c6a4ff37274bb35d4c5743335cf7f0cb3d0ef +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..407b05efb9cda1d7c029ceca5bb22db86f4d71c5 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:370864d14b359ccb8f563d8bafcb04d8bf9a6b5cb4065474c26358a6e0a20936 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f60528607e11de2fea445f1df0f4c520578444e --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5607bcb5fee4429136a494bd74ced0702727a7c37fbcd9f6d5a4109d19732cf +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3b3c01610e31ba6b7d9776e7ad7972618f90eaf --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99cb929f885b21f258221c722b79ac1887e26e32f9353e2b6586b0991f029191 +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc7f9d2545061548204d0926899042d97872eba1 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12816a19e3051ce4f892edb61ae30cf02dd88f31ff8cb7455195da564be8c8d3 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52fec99e87129538c882458d28d14f6414e022a1 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be80f48291000bd001f90c1cf6df6a6b50fd932e868ba880bb226aca21bf1417 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..524f6af839fd9351ba71f02824ae8dd2fd204a56 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12fd3b9422479211c78b4e3d088706abc330c748d9e601aab8ce6936fd2e596b +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9e70f9574c33c9cf55c07640f654431644ed5a0 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:349b042885e34d2dc8f9a370f0925e4c846a74d2c9ca4e15daea5a71e0a82644 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d4ce1c915552588986d3d2c89fecabe2e251299 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db8b5a7e1cee082e64453b2b02cae466fc4c5b85b766409b2891586dd289fe08 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08a5270d8cbe873e82407a10b409e6d6a8749b0b --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb91115f74c90129d42cfede60e2a045992361556f4176ffbce50d51eed87532 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8c6c9aaa58160b22532fb5e41cc9b8050873548 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:047e7007d2ae5f727a1d2eb3bf5fac95c37cbe74d05d33250af9d3fd399d7e86 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d498eadc9655daf1c159929d9245e3ce5b3b8c39 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42cdb6aa62d68a5f3589434a933d1691b76c41443ca065167a977025149dd450 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..82875f44c768a707529ede5bff6cb70379dae952 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16280546c46dd9047e839107b2e27eec69b267de5fef3576cc8b8fffe208b49d +size 199058647 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d6b458b7a18383e2a6ca3e45b2e373cb4e7a097 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62e0e344bf1d405bb85b00162eeccfb62cf59c4ffb4717a8990593a2b323866b +size 199058647 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d00959a63a6c7440eb7333eab34290982449963e --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0914f582c2cf2a661f68494a8d4f8e33417460304fee7ddffa1d2f8f5a5a965 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40a4ec5cbb8f41bfb9b720149cdcfbb5443cbabb --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74b02ffe320f5547c46f1cc1b1823d0d77d8578772e45800a3eb9b1f97b81920 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..021aa8741611ac667627fe55ae2e9b696d7d4fe5 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f8bcabfe9005a0a6b97b4471491ad942effabc05776825d34ae38aec0759a07 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f1f34b6c8e56a817c2af4c8ce27f88b7542b8bb --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7df6b3e631a3be233ca944cc3b25cf8c3cc2aa8793e6466fd6177440949cd618 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3377dcc38e5f5690d5830ae5d80c0b3ca3374daf --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9db3a13aa0ddfc352a822c5c12405ec3cd2cbb3bec9881115f8c76793a047f58 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef84ff04220e39fd238de2f801ff9c47cf46eff2 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12d13a79b1c9567ec6d46f6f4e7f5ed93eb45f237c629bfef029324e4754cfbe +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e712f81c082d49fd90577d8290ba275621b39fbf --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:621a053f9d3f7f9ee11a2366d2e9d1d6635f556b84855f70f85e516465e7df1e +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..027efa2d1924e9810b7b24e47fe528b5beb2405a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e11f97306526df42b0f59ad7fe50fa47a8178ca59047da124456b1ff7d0612f7 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7da5f6c44157b3633b717adb3fe1fbd53c856515 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca5ee22a94bb19565c18d27a3fa17235b6752f12dc10fcc52e463c09ca610f92 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2ef56dc7911508abcb2d00ad96dfac08b19bd23 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8564137556b3bf2489f5b2b53a69cdc595f8d53e1f085f82f5278d4f5cdbd84f +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6ce43204865f748d9c5e975b5d1fe05214eec6b --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76a6390f3fe1c38db47a1fcecf0ff73a27e75c23d665d7260170600aa3b02a3a +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba0f51bb226149c404383012ca96ac22c7469af8 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:276e0aa01c8e1b716058b935d0bc1a43bfc6bfc645aa202a29417de29379e0ca +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7566c7b6bd618bc08711b8d8935c93783064ce6c --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66a26edf288fe46a1b2ea50590b6030b01f70a420706fb9ffde5b425cfb0dd3b +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9af15478a104289ab3668fac540a3b0e307eba68 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72d7c13d66c770bf683fa552f6eb510466390c0cf92262a7d14e2c22ccbe697d +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb8478fa58aeb953e9137365daac8164443a44af --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e2f6c8fdff7bef93c8866f353b38c49c4cbe9f2138a166b0970c5bda72ec02 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5744df9f9de842904726b854821129818a18ec6 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c29564c2449e4abdd902b367713ff29660c3dda7782dc84c5c726d2499a3c92 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c972792fb22c08a81869922aa2d3d4ab6bcb210 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00aee10bcc72bbb64c0820f29ff9220dd59b1eb027767b8d19061019fb9f89c0 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..34c1ec3b9d7230159215701b8aa0ca4865d12671 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a48530bc51b8f42833239905c294811bd4be8211aca40a8c39d8b0648c69f9fd +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6610036163ea693770a70f6c92f1e1045643016b --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b42dd77cc4896bce336f23390da0de4deb013cd91ed9cdf6dfb3c663db0c7edd +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..272e608ff43574f9357c8c0903e419f9cef68c78 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a21517b65e0bfa44bb96f417d032707ec2c74713b4dec5cfbe02c63aecec9aa +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd7f47e082096d4c75277d0784f1ac581bb034a3 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5b49d7a27fb82559c0c6cb65c1dcfe020acd6d8b1c745112a4d65b51109066b +size 199058647 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..80051fde6924debf252de5bd988805ee08dd1e9d --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2cf8004614c64272ac477bc36d26b48d2bfab2d3660181ad2ee944e781b6190 +size 199058647 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..33d7c8524ca39a3cbb5d49832ed932cd3b368344 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7741de611dcd79db672155939e566c4e52923cf634e9f11ca2a95d12084a9008 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..addf0db1b1abc4b4a5a342bec939949480e9a867 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:521ffe3e400e5a126392e216e46fbce994811bc29c07475d84d503ce33d08515 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7da1f9e34690dff9c57934eca9891012859ebb94 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fc89cd37347c5b9d0a71ade2e06cfcda9fe3a057b251f434d378734e0f0f4b4 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2aaae0219ac0185b240d0a2ed47fb850d204ef6c --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4de4c7707141530475f04f8492499e63faf66e84ecd22da40954f68232fbab4 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e94e6ae29a1471af37b1efa84751edde6194740 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ab392d0011d15dcc77abc46d2325a7ccdc7f28566218025e66dcedd8c85b86a +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ea5cea7d30005c4a287ded31e691217ced8c635 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3819bd6f93498cffdecf4024d5d7704b408f186612864f88099081182d1bcf7b +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee7f189db7b3252335c835e57d28b28118afdb53 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8d49c955335b7ba6ab162ed7704811c9c5c86e256f5c57a4a5e6f1ba9edf8e2 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6780f6c2190d1bd286d6959e8b8e06ee7f1abafa --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e14209ec30fa0c62cc54d31a25759c1ad2e0a9a792bdf6926246e158504bc522 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24eaaa8dbe007107aa899922f4c6a208f4a19a66 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5589b6b4533a44a20af33c5d1c7cc208e9afdd61c56c8e0654d2de0ce09b29fb +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8d3e2e4c2e9c38e39b0c5e812ce54fdfb4a3278 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afd133b13444adbcd155b01f854d1d9bad854a6466f0c9e757af3b40cb20824f +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5e994b34da499e79ead929f9c9419ff9245d26c --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef10f1246b9ae04c7c0b41a748c472d0ca6916ff17434a7252fa004c8492aac0 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a6e0208a7585d739f21576e937e5509d0df359e --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aed2339599a09ac2d86b17414a06828396e3295f99dd34ea0382a09ef711ace4 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf053455d2d8107c85baaa65748c1ebe2c6528f6 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7af8c1dce2f0f6e6fb09e40fdc375ec7ed69ca643761e104e7d7994952cc1fa +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f95c14c22d4aa4b4b71a5a15c6a4ce9d20bacdbf --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:441583dd327f7475d3a8dd108f4b821d39bdb6ff21e02d2c82fc8235058796b0 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf3f811ce2e8d213ccde16d82227feeaed429ea7 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd649c482c97d048c2a396af3e05a2422296767c44e01ac6f9fa6effd7e24ce8 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff81edd7ae54641a6858c3eb0605fbc022c0a74a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02b9512f4c41865010c74a98d024c63812b18fa7a08230574002b3ee9b55402d +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a49dfa50f8cad667d405628db313fc71c29cee3 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6f3e2b60f73e6c8ca735dc61e3eee1ef148f768871599e91032d4fc97185b7a +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f15175384eb73fd060b6e2f887d96f23b72e935 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85ed280d43b91249baca09b5fea07d8c18d8e3435b2986ee9cdc1d0038fd7a34 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..164dd5f712b88dc5eb7eeb85ca2319c5e38cdf32 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b5d0e7b14cf4768f5907e48eb3d6082ad63c9a74a37a4591857a2e64d693e27 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b25ade2e15ec2bbd3517e45e52589000a65948e --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e81deb7361d9bd34ec19840c2fdd5235c6e8092a3bc245280d210c00d519573 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c896eb527257bdb6460cfd54776196c5e8fe39a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69dee0de123f269d93495f468c5e697acf803c70bea61b043a9fd58755217762 +size 199058647 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2090fe12849b1890bbc3e0fced41d0b49b3ee3bc --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c823b2ac9b8b2ab0a7a1dd69f63dc30d44d70c7ee5cd78786dde88f7db6e35f +size 199058647 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb3fc256da998f83b9dc6cef1270c16985a9f6ee --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad463a89232576e005fe95c1a7c0569537b485ae3a35d4636ca21b820460d06b +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a0f40331c93785c82a8f95962ba719c8530bb31 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e91bb78e6bd47a5864c9595ae945e7b7d8fe074a2821e8d92f80e538d3fc1590 +size 199058850 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb27562bb770e69348637f4897d82d273f0e2b3a --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8d7774424974872f3cc4ff44153a03010a60f9ac95002b8cd7353807ea6c76e +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..210248467154f866ce8208e82d923e8606c474f1 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:866d0c472d9a4736c2303310caf4eb469e9c5e44136d77f425895d29f9853b12 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..555e0e7d65174fba6cfaa36901de3ff53f5afb73 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3ffd166b4509a6d47e7e9c7d0ed3071e5a52c2b855fe94720f2460d28b003d3 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..48b2a0f9f893b5be6ae400784059a9048e316d76 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6e1ba007353895b17f3da8b9ef95622e1c1a875fea2aa0e6ef6f94787db1fa1 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..302ad302297639b25925268b0dea03c55ec47a00 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2798a442a916c150d223fe296af5385744b17a82bca6f44a25e1cc362d276f1b +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f334b110839460f9f37487d84eaae3b05fe8deb --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1f18608bcef1bb647708fc64ba0634b85784e3f2f4f33f955fdace829b874a5 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7451b3d4e6a7fa1c047eb977af904b3568b3c7ac --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c6175cbba35b59b0b3c4ccddc2ec8b12257c3add25ce749496baaa2fea77c9f +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6e1c1d6302dbc870942dc3c6afd78068121d265 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03d5f1029c56c5512a1b31b19299aff7fe6d91766b8650c2d2eb0d060dc218ea +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..77a11056aba0c6a37e7eaa77d4baf37dce1acead --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea687c6ae0f33124de42ce248e709ec886509368a017c7a556c121409acb4cec +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..642765ebf7680223a18b3aaa793797460fa002a3 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d77df5c67100971447c52aff35ff4f812ab2f98e7da02bc6dfd86b7d4702ed8 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8df250861de17f7307bcfb4534d2ef8c52f12166 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d87b9fc9c65e949c0bced68b0b1d23aec6faba0da6f35873755d404817c39da +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e850a5723a5644e8c953a462f8e9be250bbe9006 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f54f3b6d8ebe462be9be5a17cdb2c90aaf9d4413f4fc146cadae7229061d4747 +size 199058722 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1f52077026dca96c84307af6b1ac2530938a934 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78d2e37f4a352f96230a8a9b71dec3fd92ba4bb987f080d830eb41df84f6f0c4 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..68cd801ef10670c85944a29312c3e72f8a90f4c2 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e66929fd4f411bf4ddfa49013f89261874406ed359a1e212e403f26b0feb619 +size 199058786 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f4e9b6542a61f36cb5962c1c3c36744555c9e29 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87be75200feebcca08c23c9d997249fa47ab401630c3c1edeb8b06d24c5321c6 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe22336e41c5bf3907f480d0b2a0fcc013e60e61 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47badef69e48455f3ac881ac3002c70c76d14a6a6b404e9b53a361491c10850d +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f657bf1aaed26e084eef8d7c07bafa1e34aeb3ec --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1312d8b4f1c7cc8df27754fa7849f162e5c9eecc9c9cdbca9bf18f69407a52a8 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..101b2f60003131f3bdfe18ade91e46de532f2782 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ec927a128660ca3ecb7be4805ae001ce408de659734293026a8e2b20ebb9345 +size 199058658 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ca2a8cb7a9be8f473b4e8887ddae1f453eb0837 --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c514c89b22f1aa29e5b502e70c72f68efede7002ff399b0b126113da2338c101 +size 199058839 diff --git a/perplexity/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/perplexity/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35fae67776fa086930922da623b047937b1918ae --- /dev/null +++ b/perplexity/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d9b169aefd2ad248dc470019f58e951d5f3437a8116ba6019553fafba781faf +size 199058839 diff --git a/perplexity/global_step80108/layer_01-model_00-model_states.pt b/perplexity/global_step80108/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92e0189e3522f3dedbe4c47533ef7ea4777ae062 --- /dev/null +++ b/perplexity/global_step80108/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85df2b70cdef855639b5edc7cc3c8a2f62aa922037a6f40bafe6887edb15c6f0 +size 167511299 diff --git a/perplexity/global_step80108/layer_01-model_01-model_states.pt b/perplexity/global_step80108/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d307b309f78752995889f3d2507149b0b2f02a77 --- /dev/null +++ b/perplexity/global_step80108/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:575dcc1ddd5708016db4c903a1968e15297a33bafb898224f0ffa5e5225f76c9 +size 167511299 diff --git a/perplexity/global_step80108/layer_03-model_00-model_states.pt b/perplexity/global_step80108/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99c275d68c4c61567dd2252f144415dede4ea28d --- /dev/null +++ b/perplexity/global_step80108/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3628c6d5770f14f314d3ffaf258e85abfa04f3f0c465230059aa9923c1e49189 +size 113308931 diff --git a/perplexity/global_step80108/layer_03-model_01-model_states.pt b/perplexity/global_step80108/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a18f6eacc013fead5030777579d3204adae254ff --- /dev/null +++ b/perplexity/global_step80108/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92a2bfe0d68cbd09b4b8c97b65a2e052df2df02d21f7dae3ae0638a35c5f3be6 +size 113308931 diff --git a/perplexity/global_step80108/layer_04-model_00-model_states.pt b/perplexity/global_step80108/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb7ab0e947eedd0cf692809ad8bcef36f0167475 --- /dev/null +++ b/perplexity/global_step80108/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b962a234127df9610a549d350dab6cf48ad16160e5eb57396a8ae08c32cdcc1d +size 113308931 diff --git a/perplexity/global_step80108/layer_04-model_01-model_states.pt b/perplexity/global_step80108/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a93dc338bb55189c1bac209bf87b38d5a185a718 --- /dev/null +++ b/perplexity/global_step80108/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21237b8ec3264547429ec87441b249bce601b1bf0d87e3b35a3ee2e76a870e98 +size 113308931 diff --git a/perplexity/global_step80108/layer_05-model_00-model_states.pt b/perplexity/global_step80108/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3286a537249890a6bc76898adc548978219670bb --- /dev/null +++ b/perplexity/global_step80108/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ceb2699923172197248721678383f7795d3a4359f3e42137ea20f0a1b65c384 +size 113308931 diff --git a/perplexity/global_step80108/layer_05-model_01-model_states.pt b/perplexity/global_step80108/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..45031c019b4d3ff899ea81feddfc58947e8bf6c0 --- /dev/null +++ b/perplexity/global_step80108/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9431a2b2c9922ced38ac10792d4782c842888688b6149a5919338b102540458 +size 113308931 diff --git a/perplexity/global_step80108/layer_06-model_00-model_states.pt b/perplexity/global_step80108/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1cd15205c39f3e9327c4171a96cdd81139640da8 --- /dev/null +++ b/perplexity/global_step80108/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c73e941feb8ced027235d7a5e4f7c81f9c6930f8c4596ae567d54421c598d8d4 +size 113308931 diff --git a/perplexity/global_step80108/layer_06-model_01-model_states.pt b/perplexity/global_step80108/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..302d8c01add0b427b302eaf1d30ad499ac3cfb2f --- /dev/null +++ b/perplexity/global_step80108/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb115b3111abc4ebdefbad54fd402587c39a18d82d2ee8ac6f495a0b1882b7d6 +size 113308931 diff --git a/perplexity/global_step80108/layer_07-model_00-model_states.pt b/perplexity/global_step80108/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..226bca0a2cfaa9e68b238ea7fa7fa0dbe2fe92a8 --- /dev/null +++ b/perplexity/global_step80108/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b441fdd15e08c57c9719acd2906a3bc97881391fc1666bc4e0ac896abf8b3f49 +size 113308931 diff --git a/perplexity/global_step80108/layer_07-model_01-model_states.pt b/perplexity/global_step80108/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5bbec16ac0f01e65521586f5cc170d8052c2381 --- /dev/null +++ b/perplexity/global_step80108/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0eab6d603e2a3d6f1026af40073a676aba3e49b8a4b3811485652c561013d44b +size 113308931 diff --git a/perplexity/global_step80108/layer_08-model_00-model_states.pt b/perplexity/global_step80108/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3363d0098fbc3cb5d0bcf58ada59b6563295a3a5 --- /dev/null +++ b/perplexity/global_step80108/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7759125afcb867969dbd2f52f807fa6b811b650c1673aa694d320ad8db3c1eba +size 113308931 diff --git a/perplexity/global_step80108/layer_08-model_01-model_states.pt b/perplexity/global_step80108/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..272eb68536d4aab028c18e07257311d0b65d787c --- /dev/null +++ b/perplexity/global_step80108/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03fcaefe3c3ea6591420a4dfd34e8037de3dec81c375afad595f0299a407bf01 +size 113308931 diff --git a/perplexity/global_step80108/layer_09-model_00-model_states.pt b/perplexity/global_step80108/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..68fd4d10e9056c4d8334eab910f218804db83d6b --- /dev/null +++ b/perplexity/global_step80108/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ecd333d9930973ee8e017e1cd5d5412b99c68b29360f59cfe4ca8d1adb84b41 +size 113308931 diff --git a/perplexity/global_step80108/layer_09-model_01-model_states.pt b/perplexity/global_step80108/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf59cfb6022a99c105271bb216357419757e8440 --- /dev/null +++ b/perplexity/global_step80108/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca7bb721aacbba443b1330fc0cbee4087195efa98fcb77b640c220d159b44c55 +size 113308931 diff --git a/perplexity/global_step80108/layer_10-model_00-model_states.pt b/perplexity/global_step80108/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..195ac3d1de1ffb14f606d6a177a10f44be5b1b53 --- /dev/null +++ b/perplexity/global_step80108/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c8abb6fc08a8a947c63ce1ab5d816364cd66770518b85f4dcf747170662dd84 +size 113308931 diff --git a/perplexity/global_step80108/layer_10-model_01-model_states.pt b/perplexity/global_step80108/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a78cc793c33c1d73e290ccda894ff8a5a27945c6 --- /dev/null +++ b/perplexity/global_step80108/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:642ff90f469b49cf671b9d2cdeb9c842c0cdebea2792a608cf229b5483b83675 +size 113308931 diff --git a/perplexity/global_step80108/layer_11-model_00-model_states.pt b/perplexity/global_step80108/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3bcded8b8fe448cc1849d8beb624c92f7fe663a8 --- /dev/null +++ b/perplexity/global_step80108/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bff89061ce5f9e18611838842c6bde8dd71a34928355d9212a29ca9afa4eead +size 113308931 diff --git a/perplexity/global_step80108/layer_11-model_01-model_states.pt b/perplexity/global_step80108/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be4a0468bc92616c697e88a8c8d275c33f109556 --- /dev/null +++ b/perplexity/global_step80108/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f307b9268f8bcf49879787fd40d44c0c0112f7cabb0f9ef18840260f5e370ff +size 113308931 diff --git a/perplexity/global_step80108/layer_12-model_00-model_states.pt b/perplexity/global_step80108/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6a5d228a7cb8fa8d54f6a6ea81921385213fd88 --- /dev/null +++ b/perplexity/global_step80108/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3b8838822c26025b479eee176901f6c63b022a912a40af6c8dd950b1ab0f879 +size 113308931 diff --git a/perplexity/global_step80108/layer_12-model_01-model_states.pt b/perplexity/global_step80108/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..570255ffbecc499eab74611b96da0a1c254b6cb6 --- /dev/null +++ b/perplexity/global_step80108/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7caa32c3807a75013a1839ce6ba6d6c23d360a18611e6c7b56aeb1c9098b5f5 +size 113308931 diff --git a/perplexity/global_step80108/layer_13-model_00-model_states.pt b/perplexity/global_step80108/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d03d7b99ca9038429926ab2fc49fd410f18a5e77 --- /dev/null +++ b/perplexity/global_step80108/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ad253b7f600f89fe9e2cc86dc2611892691d339bcd8380f1fedd4381785ea34 +size 113308931 diff --git a/perplexity/global_step80108/layer_13-model_01-model_states.pt b/perplexity/global_step80108/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d10b3adada9beb59f5ad956141c09e6ca0f88f70 --- /dev/null +++ b/perplexity/global_step80108/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d3c9941644a11055dab881ffffbe97bc313e195e3cd0afbe6cb9ac1b5def997 +size 113308931 diff --git a/perplexity/global_step80108/layer_14-model_00-model_states.pt b/perplexity/global_step80108/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7b1b050248d8821451d0d71135a3a618ba0ac18 --- /dev/null +++ b/perplexity/global_step80108/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6e2640d35e50d13f869c141cafa9fa7609d7bddbdae243bfe6e8d967096842a +size 113308931 diff --git a/perplexity/global_step80108/layer_14-model_01-model_states.pt b/perplexity/global_step80108/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24358aaa01fb5feeb658dfea8e0a1c2cc7d099f9 --- /dev/null +++ b/perplexity/global_step80108/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37eec7a5235e1fe87ecc9adfddafe1a3a314f20e2d2f155ced3ae668ade136ae +size 113308931 diff --git a/perplexity/global_step80108/layer_15-model_00-model_states.pt b/perplexity/global_step80108/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec3cd600b576623e966bbfc29cf51ea7aba02740 --- /dev/null +++ b/perplexity/global_step80108/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f093fabac3a74a0bc5b178f58730772a9fce96849360ae677a99ad2dac561518 +size 113308931 diff --git a/perplexity/global_step80108/layer_15-model_01-model_states.pt b/perplexity/global_step80108/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7c1e8170516b8aa2fb6a6258d4312bb46b3a862 --- /dev/null +++ b/perplexity/global_step80108/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3ce30972af94f0973c893252c3cb660209ab2673660a7d9b9643aad5e824653 +size 113308931 diff --git a/perplexity/global_step80108/layer_16-model_00-model_states.pt b/perplexity/global_step80108/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38b91a5a69bbd7ab1d35fbcfe14f4cff58083765 --- /dev/null +++ b/perplexity/global_step80108/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52dd8a1a3f29e7595c7f3305e54d4b453af6b29f5b0402e0ff3b624f56191749 +size 113308931 diff --git a/perplexity/global_step80108/layer_16-model_01-model_states.pt b/perplexity/global_step80108/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2241c02e66b736addaabc6420bf709603c7d3d42 --- /dev/null +++ b/perplexity/global_step80108/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46de294fe0911491ff277aa4b8e5420912dba57d2acb043e9b7fcb7cf4afa718 +size 113308931 diff --git a/perplexity/global_step80108/layer_17-model_00-model_states.pt b/perplexity/global_step80108/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4681a31252bd8bc0ea2aae1852ceb03b384446f3 --- /dev/null +++ b/perplexity/global_step80108/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:142a3c97c885c8e8d6b9da572e39cd62526a2ee8220a6521b0af5dde20b6f0a9 +size 113308931 diff --git a/perplexity/global_step80108/layer_17-model_01-model_states.pt b/perplexity/global_step80108/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30c4009d6865894ba5d602907af1bd49bb0c234e --- /dev/null +++ b/perplexity/global_step80108/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebd25a4bff731b7cfd8d15c2876656052e7f89422e35d1344da60942cbbbe11b +size 113308931 diff --git a/perplexity/global_step80108/layer_18-model_00-model_states.pt b/perplexity/global_step80108/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..978b8ea319a54d70918519e39de73a3705ce313c --- /dev/null +++ b/perplexity/global_step80108/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fac31f8e8916f565c78f1df4523e98704f9868787be83d73e4397977bec1dc51 +size 113308931 diff --git a/perplexity/global_step80108/layer_18-model_01-model_states.pt b/perplexity/global_step80108/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f6c7d6b8ef525aeddc991f0f0a1682a6839889b --- /dev/null +++ b/perplexity/global_step80108/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b381f577632e3b9cf8a03bcc5f4c34d4f5361e27edb2c71b1fd617f14403fa92 +size 113308931 diff --git a/perplexity/global_step80108/layer_19-model_00-model_states.pt b/perplexity/global_step80108/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce6e84288ae3b350752a565eaf8e3f5f413596fd --- /dev/null +++ b/perplexity/global_step80108/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d93252c30dcb8d3de1e09ed114c50e64f313964c4bfb34c3919b9f8e534bc7a +size 113308931 diff --git a/perplexity/global_step80108/layer_19-model_01-model_states.pt b/perplexity/global_step80108/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8af63bb428b013def772e4352bd21d0bad8951ac --- /dev/null +++ b/perplexity/global_step80108/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8a365f34d417f6bd53c78e5e0ccf995a74700a110bc7d2ca0a2fe494930b74a +size 113308931 diff --git a/perplexity/global_step80108/layer_20-model_00-model_states.pt b/perplexity/global_step80108/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..420ed91e210a32122ab24d73fa26bf028bbd8442 --- /dev/null +++ b/perplexity/global_step80108/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c86ca6f25a5039ca8e34c4a28dea145ed7d9ea95161ea4436a0c6409d1d44daa +size 113308931 diff --git a/perplexity/global_step80108/layer_20-model_01-model_states.pt b/perplexity/global_step80108/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6236f3bf781741b2967aa1883bdc2f2a334a7172 --- /dev/null +++ b/perplexity/global_step80108/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6f46650389b705ee5d9d5cb06b6deaf32a1ebc1d99fdcad2e45ed28e72cf3d9 +size 113308931 diff --git a/perplexity/global_step80108/layer_21-model_00-model_states.pt b/perplexity/global_step80108/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c39a1a7f3fa8655b358be12ee9c657c4ba687648 --- /dev/null +++ b/perplexity/global_step80108/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23ba04bcef74c4a8e4b62ba9cf962d70006ecb963ac0f60a3596a0098af61cd6 +size 113308931 diff --git a/perplexity/global_step80108/layer_21-model_01-model_states.pt b/perplexity/global_step80108/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd4b2b6898b62b0b502826a3ecb20a7d0197a777 --- /dev/null +++ b/perplexity/global_step80108/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74b4c51cb21b45ac21c40f86e394bc5da9aa044af9aae97c50f5668bcf35be2e +size 113308931 diff --git a/perplexity/global_step80108/layer_22-model_00-model_states.pt b/perplexity/global_step80108/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe8a424222260f30500b442623e54f3848d91288 --- /dev/null +++ b/perplexity/global_step80108/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34e4fb56d018f146b76ee526cf26bdb4310bc2fdafc7701307d1c06b8faa7379 +size 113308931 diff --git a/perplexity/global_step80108/layer_22-model_01-model_states.pt b/perplexity/global_step80108/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..595f6642a56eb5f9bbbec460c94efd740070d249 --- /dev/null +++ b/perplexity/global_step80108/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f4c3c6eb146aff97f584700c8435d58495f80a2fd81e900b3dc37d500e3f02d +size 113308931 diff --git a/perplexity/global_step80108/layer_23-model_00-model_states.pt b/perplexity/global_step80108/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2c300e1f3c4deaac5e495378c796bddbb14ece9 --- /dev/null +++ b/perplexity/global_step80108/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e963c81ae66339f7635aaaf159e405f1e7aeeada0d54b92172737e9c27a88b90 +size 113308931 diff --git a/perplexity/global_step80108/layer_23-model_01-model_states.pt b/perplexity/global_step80108/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ffb8dd4dc2b7051a0573c2206d909e366e925c5 --- /dev/null +++ b/perplexity/global_step80108/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a3fc6b7d91035383e50d7ecc155454371d0099e61fbe9a99b94533e59e393cc +size 113308931 diff --git a/perplexity/global_step80108/layer_24-model_00-model_states.pt b/perplexity/global_step80108/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36a7c673aacbf6e0262336c931ffc39a6b9e8f83 --- /dev/null +++ b/perplexity/global_step80108/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b84255a4b975a8328dd9cbf9d76769bdee1d7277c06bbf7ee7b07317fb1d5b95 +size 113308931 diff --git a/perplexity/global_step80108/layer_24-model_01-model_states.pt b/perplexity/global_step80108/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcbc91c7d3423de57fb385c65b3d0e0f273a935c --- /dev/null +++ b/perplexity/global_step80108/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06343b11656f76dffa924a681fe0aa81725789925ee93ed05ff0c7415f7f8409 +size 113308931 diff --git a/perplexity/global_step80108/layer_25-model_00-model_states.pt b/perplexity/global_step80108/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0aa3dfb5f300591a3459d4956573d8634b6e82c9 --- /dev/null +++ b/perplexity/global_step80108/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbe07c75fb26af8f6dd3cba6186ca00b6fb4246e374a2d1f0b8d7dd2acf4413e +size 113308931 diff --git a/perplexity/global_step80108/layer_25-model_01-model_states.pt b/perplexity/global_step80108/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dba08d425f0f2bcc1ba0788269a584b051065729 --- /dev/null +++ b/perplexity/global_step80108/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90ea03995249aa4462eb6d4bb56340081f1aa1cd35eccb22b6bea0ec43ec6ec5 +size 113308931 diff --git a/perplexity/global_step80108/layer_26-model_00-model_states.pt b/perplexity/global_step80108/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00011a4b767008a6e5b2aeb3587cfc8cf36cce4e --- /dev/null +++ b/perplexity/global_step80108/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4764482861223d661bcfe2a8dc3495bfbb62330c4727fc3b3b497ad9f6a0ddd +size 113308931 diff --git a/perplexity/global_step80108/layer_26-model_01-model_states.pt b/perplexity/global_step80108/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a57aa06ba555eaa7d7bada51e97bb2f6bf0b53e --- /dev/null +++ b/perplexity/global_step80108/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fad9b9cf9a6bc06c9bfaf2f290316aaca55d4b4590a004c8609f5be1cc610879 +size 113308931 diff --git a/perplexity/global_step80108/layer_27-model_00-model_states.pt b/perplexity/global_step80108/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9af2fdfed8646ddd248e911e40b1c9f9c826f229 --- /dev/null +++ b/perplexity/global_step80108/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:666e20797395f24d125bf8b06c90b92a13621a071dcce7ae1950a96beb24956f +size 113308931 diff --git a/perplexity/global_step80108/layer_27-model_01-model_states.pt b/perplexity/global_step80108/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..68ad228c8012d83be198fc89e1f4c4ec59623e3a --- /dev/null +++ b/perplexity/global_step80108/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74bfa351f5977b50c7df9ab5bec8531d5090e5102b515d6152a5dd428ecc5104 +size 113308931 diff --git a/perplexity/global_step80108/layer_28-model_00-model_states.pt b/perplexity/global_step80108/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bcd9a1ce9bf162fdefbf99d3a891fc65832cc5ba --- /dev/null +++ b/perplexity/global_step80108/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed3987b2b57c0be11361c93421308480adf3d3e30a0348ce5a3872312e3d3952 +size 113308931 diff --git a/perplexity/global_step80108/layer_28-model_01-model_states.pt b/perplexity/global_step80108/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c886ef6962644fff6d761ef589b2eba8b15f960e --- /dev/null +++ b/perplexity/global_step80108/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:314358ffb61017e1d956a3907b5842ac88aad39f61a2de18b94b192d4d413130 +size 113308931 diff --git a/perplexity/global_step80108/layer_29-model_00-model_states.pt b/perplexity/global_step80108/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7aec373ade9ac0714d3b33ed4b08245e6465a84 --- /dev/null +++ b/perplexity/global_step80108/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70bf8c87131be1544b65cd6886b51103f8ec4e17725ada7b7179074059fbe88d +size 113308931 diff --git a/perplexity/global_step80108/layer_29-model_01-model_states.pt b/perplexity/global_step80108/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..023040df16dbde9d7d26556441b24aa7815f659f --- /dev/null +++ b/perplexity/global_step80108/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af85c1a8b9bad46d5d6d95c64741c0624a62ece5114802d0bd786a0431ee71dc +size 113308931 diff --git a/perplexity/global_step80108/layer_30-model_00-model_states.pt b/perplexity/global_step80108/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f1ff1551df6b910b47f50846bdd0c39d7852731 --- /dev/null +++ b/perplexity/global_step80108/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd8aebb2402e9577bc789d5df89483478ee982992bc5bc44ec4cc72bb178d725 +size 113308931 diff --git a/perplexity/global_step80108/layer_30-model_01-model_states.pt b/perplexity/global_step80108/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4ad2984fd838ade49a6cd5de8a1e9504a456bf3 --- /dev/null +++ b/perplexity/global_step80108/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef67b2910937af65e3d0433046577e406a62ecc44cbdabe4254570a541b9b234 +size 113308931 diff --git a/perplexity/global_step80108/layer_31-model_00-model_states.pt b/perplexity/global_step80108/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..debbe5aa66ed3072ebb0ce3d384807e6637aff90 --- /dev/null +++ b/perplexity/global_step80108/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e7f2c0a521a3a54531777adf73cf345cdd7429f7e4aab014d84630b7a2b93b3 +size 113308931 diff --git a/perplexity/global_step80108/layer_31-model_01-model_states.pt b/perplexity/global_step80108/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d3f5a6f21a5fe396c4fa28edbeda80da59dcd69 --- /dev/null +++ b/perplexity/global_step80108/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f70910eba77570c3be0e7721ea2cde2d2f750cc489d73f35138ed2fcdea6a3f2 +size 113308931 diff --git a/perplexity/global_step80108/layer_32-model_00-model_states.pt b/perplexity/global_step80108/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03e1c98ac8119f0c499a81fb19a1b79d4586ba79 --- /dev/null +++ b/perplexity/global_step80108/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e70c39499e6516ea06da507f31d5daf813d50b22bfe0c0f9e24e6979583f5650 +size 113308931 diff --git a/perplexity/global_step80108/layer_32-model_01-model_states.pt b/perplexity/global_step80108/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..102db7ad7435011294a0999f7806cf02ea53aedd --- /dev/null +++ b/perplexity/global_step80108/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:093ab26d8ba31e40e2c28a3ed63afab11b661c896ac75033256ee443f997dcc2 +size 113308931 diff --git a/perplexity/global_step80108/layer_33-model_00-model_states.pt b/perplexity/global_step80108/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dfac7df9a5080446e29b0ec21e96b14f51ae0847 --- /dev/null +++ b/perplexity/global_step80108/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b82ebfbc3d7df508e656d5d05be4f332f0922ed97108c84b49e28046b265eeae +size 113308931 diff --git a/perplexity/global_step80108/layer_33-model_01-model_states.pt b/perplexity/global_step80108/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..25f9d744a1be74be8b624c94a7690ea7442cd50a --- /dev/null +++ b/perplexity/global_step80108/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:933102c5b4e90461440e18f517e0a82ded6396148c2dfe6f140e04b563a6a620 +size 113308931 diff --git a/perplexity/global_step80108/layer_34-model_00-model_states.pt b/perplexity/global_step80108/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46d2030cb06760d3fb6e1cbeb265bb3ff98fd339 --- /dev/null +++ b/perplexity/global_step80108/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b082ddd33d095cba84000256d14976490aeae79f1d3b3ce8551aa1eb749764a6 +size 113308931 diff --git a/perplexity/global_step80108/layer_34-model_01-model_states.pt b/perplexity/global_step80108/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ae5948f78f9c3f0951bb191cad9d6ddb210f54c --- /dev/null +++ b/perplexity/global_step80108/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:792bb88738a21aa8ee12b9ec3502e457a606a8b9720c283344bb435535283e8c +size 113308931 diff --git a/perplexity/global_step80108/layer_35-model_00-model_states.pt b/perplexity/global_step80108/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c00384c335902a4a4f4b7e902727111cd80ddeee --- /dev/null +++ b/perplexity/global_step80108/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:349afa210db26c87c7bf390c8e19b559ae9f0cadbdef50b64737c4c6e0b2cc5f +size 113308931 diff --git a/perplexity/global_step80108/layer_35-model_01-model_states.pt b/perplexity/global_step80108/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d53380f5c6f9dd915a9ac1f34523f878dc10538 --- /dev/null +++ b/perplexity/global_step80108/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88dc5ec2e401e7a5720327e51bf8ade857ea96299d885375ea8a58e078f46815 +size 113308931 diff --git a/perplexity/global_step80108/layer_36-model_00-model_states.pt b/perplexity/global_step80108/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfd9e8b0f343abf7d2dfb22797ab4f3152224b03 --- /dev/null +++ b/perplexity/global_step80108/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf36d33a2ca646037f7e35b9d908384ca4db345c97522be64af89c30613e18e6 +size 113308931 diff --git a/perplexity/global_step80108/layer_36-model_01-model_states.pt b/perplexity/global_step80108/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c80f58a45ed8c494170ca03d067b855053997b6e --- /dev/null +++ b/perplexity/global_step80108/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d62fdf02ce2de4c5acc4157607fe40834871bc438fae12d9bab154764fa00fd +size 113308931 diff --git a/perplexity/global_step80108/layer_37-model_00-model_states.pt b/perplexity/global_step80108/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cff66041867b277e88523b4937562cb167befcb8 --- /dev/null +++ b/perplexity/global_step80108/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d37151dcd46574415f858ec139f584e6ef7b1a0b8d4009ef18a6aef7f353322 +size 113308931 diff --git a/perplexity/global_step80108/layer_37-model_01-model_states.pt b/perplexity/global_step80108/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aae5126bb8c15106aac70b3689939f38d40b3c5e --- /dev/null +++ b/perplexity/global_step80108/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:136d57c744fbe93a3b3de7b5887e8413faebf009da2281238b9c21cd66311acb +size 113308931 diff --git a/perplexity/global_step80108/layer_38-model_00-model_states.pt b/perplexity/global_step80108/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..69e27fa5c5a85944ebfbc9de61e913bfecad67b6 --- /dev/null +++ b/perplexity/global_step80108/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1843933824db23c0f539d5b1f909614185f64c9cbb59389bc3e6e11695e685e3 +size 113308931 diff --git a/perplexity/global_step80108/layer_38-model_01-model_states.pt b/perplexity/global_step80108/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8aa6255a1652016768ee227123520f315434336 --- /dev/null +++ b/perplexity/global_step80108/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e364a4745c4f00a65e65c833278996d09c9018f13b1a4184de6446e6a2a26d58 +size 113308931 diff --git a/perplexity/global_step80108/layer_40-model_00-model_states.pt b/perplexity/global_step80108/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9256121e8518fe663f1fb7fdaf77160820e1b67c --- /dev/null +++ b/perplexity/global_step80108/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05ffcb74318b94b3657b8a2d200dc15becd7f288f2ec18debf43214c5b10b165 +size 13507 diff --git a/perplexity/global_step80108/layer_40-model_01-model_states.pt b/perplexity/global_step80108/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd932ab7703e1f2eb34e950d09ce67f5099048d2 --- /dev/null +++ b/perplexity/global_step80108/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb4437c8b18fc4e9edf90186254c75dbd4405a2a3f56963eff53f9aed674865f +size 13507 diff --git a/perplexity/global_step80108/mp_rank_00_model_states.pt b/perplexity/global_step80108/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8f35fd30a8048f1efcbe9b752f218fec04c7693 --- /dev/null +++ b/perplexity/global_step80108/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4683dd0c2b975417f079db3ca6c05258e0b2b6587b1a00057bd2c0eafbb65d7 +size 51507 diff --git a/perplexity/global_step80108/mp_rank_01_model_states.pt b/perplexity/global_step80108/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2929476defa2bee942efbf80d5bc7100c50af8e7 --- /dev/null +++ b/perplexity/global_step80108/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b63b3275c1f859c4e5783845e18f01152a2942b9c0c61964810a27c029908961 +size 51507 diff --git a/perplexity/sbatch_4b284bc4perplexity.sh b/perplexity/sbatch_4b284bc4perplexity.sh new file mode 100644 index 0000000000000000000000000000000000000000..01074e94775141674f428101ca559d89fd24f9b9 --- /dev/null +++ b/perplexity/sbatch_4b284bc4perplexity.sh @@ -0,0 +1,163 @@ +#!/bin/bash +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=40 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + + +VARIANT=4b284bc4perplexity + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +DATA_PATH="/scratch/project_462000119/data/c4perplexity/gpt2tok_perplexity_text_document" +TRAIN_DATA_PATH=trainperplexity.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4perplexity/gpt2tok_perplexity_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=2 + +MICRO_BATCH_SIZE=2 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_4516M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=10000 + +# Tokens: 84_000_000_000 +# -> Samples: 41_015_625.0 +TRAIN_SAMPLES=41_015_625 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.95 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 410_156 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/perplexity/sbatch_4b284bc4perplexityval.sh b/perplexity/sbatch_4b284bc4perplexityval.sh new file mode 100644 index 0000000000000000000000000000000000000000..280fc4e799655cddb40c93f826fd5855b6c98f95 --- /dev/null +++ b/perplexity/sbatch_4b284bc4perplexityval.sh @@ -0,0 +1,168 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=40 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=4b284bc4perplexityval +VARIANT_CKPT=checkpoints_4b284bc4perplexity + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=$VARIANT_CKPT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train1b5.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=2 + +MICRO_BATCH_SIZE=2 +GRADIENT_ACCUMULATION_STEPS=2 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_4516M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 25611230000 +# -> Samples: 12505484 +TRAIN_SAMPLES=1 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 0 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + --override-lr-scheduler \ + --reset-progress \ + --no-load-optim \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1 \ + --eval-iters 100 \ + --eval-only true \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun_32.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/perplexity/tensorboard_4b284bc4perplexity/events.out.tfevents.1683712349.nid006472.30261.0 b/perplexity/tensorboard_4b284bc4perplexity/events.out.tfevents.1683712349.nid006472.30261.0 new file mode 100644 index 0000000000000000000000000000000000000000..47cb6cdbfd77ddef8c6e7d33f658627ba85a9f31 --- /dev/null +++ b/perplexity/tensorboard_4b284bc4perplexity/events.out.tfevents.1683712349.nid006472.30261.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9122b6e8da14b168d93c7f6aeb7b8adc4458e39b685e3715f5a776a6ce308eff +size 111690722 diff --git a/perplexity/tensorboard_4b284bc4perplexity/events.out.tfevents.1683885167.nid005934.43345.0 b/perplexity/tensorboard_4b284bc4perplexity/events.out.tfevents.1683885167.nid005934.43345.0 new file mode 100644 index 0000000000000000000000000000000000000000..b388bbe83a6b244e8f09fc3a53dc64c94b835984 --- /dev/null +++ b/perplexity/tensorboard_4b284bc4perplexity/events.out.tfevents.1683885167.nid005934.43345.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de7becfc03e79c2e544bae5df3f98e954194a325285a900afc3e2c09fb3b4cc9 +size 36156591 diff --git a/perplexity/tensorboard_4b284bc4perplexity/events.out.tfevents.1683940992.nid005934.114754.0 b/perplexity/tensorboard_4b284bc4perplexity/events.out.tfevents.1683940992.nid005934.114754.0 new file mode 100644 index 0000000000000000000000000000000000000000..4fabe02828c71f30dab8e252e044a4287d9d9e42 --- /dev/null +++ b/perplexity/tensorboard_4b284bc4perplexity/events.out.tfevents.1683940992.nid005934.114754.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa7894ec418ea428b61e23a75da169a087a4b10ccd7da322600204b9c062ce21 +size 16424 diff --git a/perplexity/tensorboard_4b284bc4perplexity/events.out.tfevents.1683941259.nid007233.29683.0 b/perplexity/tensorboard_4b284bc4perplexity/events.out.tfevents.1683941259.nid007233.29683.0 new file mode 100644 index 0000000000000000000000000000000000000000..83ec588b976db43f2f1a671adaa25f44c7a70560 --- /dev/null +++ b/perplexity/tensorboard_4b284bc4perplexity/events.out.tfevents.1683941259.nid007233.29683.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b0a8537f8b785c2f9d2190c49cdcc45ea52df7b68504854cb1a2ee7c2e0a229 +size 20523 diff --git a/perplexity/tensorboard_4b284bc4perplexityval/events.out.tfevents.1683959851.nid006147.37424.0 b/perplexity/tensorboard_4b284bc4perplexityval/events.out.tfevents.1683959851.nid006147.37424.0 new file mode 100644 index 0000000000000000000000000000000000000000..b0d949d3596cb7c86fa85e4f1143aa8a6ef32472 --- /dev/null +++ b/perplexity/tensorboard_4b284bc4perplexityval/events.out.tfevents.1683959851.nid006147.37424.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e98262dfb66d0ae0af3e4f4b2bf071941af79be73ace500628295055d47782e +size 980 diff --git a/perplexity/transformers/config.json b/perplexity/transformers/config.json new file mode 100644 index 0000000000000000000000000000000000000000..df814c3a71ee49044410636b5eeb4f43611690ba --- /dev/null +++ b/perplexity/transformers/config.json @@ -0,0 +1 @@ +{"vocab_size": 50432, "n_positions": 2048, "n_embd": 3072, "n_layer": 36, "n_head": 24, "n_inner": 12288, "activation_function": "gelu", "resid_pdrop": 0.1, "embd_pdrop": 0.1, "attn_pdrop": 0.1, "layer_norm_epsilon": 1e-05, "initializer_range": 0.02, "summary_type": "cls_index", "summary_use_proj": true, "summary_activation": null, "summary_first_dropout": 0.1, "summary_proj_to_labels": true, "scale_attn_weights": true, "use_cache": true, "scale_attn_by_inverse_layer_idx": false, "reorder_and_upcast_attn": false, "bos_token_id": 50256, "eos_token_id": 50256, "return_dict": true, "output_hidden_states": false, "output_attentions": false, "torchscript": false, "torch_dtype": null, "use_bfloat16": false, "tf_legacy_loss": false, "pruned_heads": {}, "tie_word_embeddings": true, "is_encoder_decoder": false, "is_decoder": false, "cross_attention_hidden_size": null, "add_cross_attention": false, "tie_encoder_decoder": false, "max_length": 20, "min_length": 0, "do_sample": false, "early_stopping": false, "num_beams": 1, "num_beam_groups": 1, "diversity_penalty": 0.0, "temperature": 1.0, "top_k": 50, "top_p": 1.0, "typical_p": 1.0, "repetition_penalty": 1.0, "length_penalty": 1.0, "no_repeat_ngram_size": 0, "encoder_no_repeat_ngram_size": 0, "bad_words_ids": null, "num_return_sequences": 1, "chunk_size_feed_forward": 0, "output_scores": false, "return_dict_in_generate": false, "forced_bos_token_id": null, "forced_eos_token_id": null, "remove_invalid_values": false, "exponential_decay_length_penalty": null, "suppress_tokens": null, "begin_suppress_tokens": null, "architectures": ["GPT2LMHeadModel"], "finetuning_task": null, "id2label": {"0": "LABEL_0", "1": "LABEL_1"}, "label2id": {"LABEL_0": 0, "LABEL_1": 1}, "tokenizer_class": null, "prefix": null, "pad_token_id": null, "sep_token_id": null, "decoder_start_token_id": null, "task_specific_params": null, "problem_type": null, "_name_or_path": "", "transformers_version": "4.25.0.dev0", "n_ctx": 1024, "gradient_checkpointing": false, "model_type": "gpt2"} \ No newline at end of file diff --git a/perplexity/transformers/pytorch_model.bin b/perplexity/transformers/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..678e7b9353e5684ca89c28c30805b06862070b74 --- /dev/null +++ b/perplexity/transformers/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1279abc3cd7460e3b7108de5a3d33f68df84b490bafbbd321903b75d8347e31 +size 8781203669 diff --git a/perplexity25/3510898.err b/perplexity25/3510898.err new file mode 100644 index 0000000000000000000000000000000000000000..c02485190a73c755581e60c869c00b238298889e --- /dev/null +++ b/perplexity25/3510898.err @@ -0,0 +1,3620 @@ +30: 2023-05-13 09:35:31.776492: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:31.776498: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:31.776494: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:31.776781: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:31.776782: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:31.776782: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: 2023-05-13 09:35:31.776688: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:31.776691: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:31.776701: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:31.776777: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:31.776775: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:31.776775: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:31.776715: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:31.776715: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:31.776977: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:31.776984: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:31.776982: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:31.776982: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:31.777107: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:31.777113: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:31.777113: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:31.777120: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:31.777108: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:31.777123: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:31.777136: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: 2023-05-13 09:35:31.777249: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:31.777259: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:31.777256: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:31.777135: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:31.777125: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:31.777121: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:31.777144: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:31.777128: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:31.777213: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:31.777225: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:31.777217: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: 2023-05-13 09:35:31.777210: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:31.777206: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:31.777217: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:31.777219: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:31.777207: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:31.777314: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:31.777303: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:31.777310: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: 2023-05-13 09:35:31.777205: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:31.777216: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:31.777369: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:31.777371: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:31.777376: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:31.777307: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:31.777309: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:31.777375: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:31.777376: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:31.777370: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:31.777378: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:31.777381: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:31.777380: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:31.777385: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:31.777368: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:31.777391: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:31.777573: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:31.777577: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:31.777573: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:31.777581: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:31.777631: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:31.777636: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:31.777903: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:31.778902: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:31.779241: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:31.779934: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:31.780584: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:31.781750: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:31.781995: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:31.783283: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:31.784109: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:31.786969: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:31.790887: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:31.793360: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:31.795227: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:31.797544: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:31.799872: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:31.801414: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:31.801699: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:31.802175: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:31.804117: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:31.805240: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:31.809775: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:31.811172: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:31.813199: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:31.814896: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:31.816466: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:31.817538: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:31.818100: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:31.818560: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:31.819424: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:31.820840: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:31.822009: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:31.825349: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:31.828240: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:31.829853: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:31.830756: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:31.831533: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:31.833030: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:31.833707: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:31.835730: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:31.838674: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:31.839899: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:31.841306: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:31.842308: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:31.844410: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:31.844417: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:31.844422: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:31.844405: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:31.844423: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:31.844424: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:31.844404: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:31.844431: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:31.847253: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:31.848490: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:31.849656: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:31.849927: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:31.849927: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:31.849934: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:31.849931: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:31.849934: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:31.849930: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:31.849920: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:31.849936: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:31.852053: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:31.852057: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:31.852065: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:31.852053: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:31.852061: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:31.852074: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:31.852074: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:31.852065: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:31.859271: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:31.859281: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:31.859280: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:31.859283: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:31.859291: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:31.859286: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:31.859287: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:31.859298: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:31.863500: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:31.863513: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:31.863523: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:31.863524: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:31.863531: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:31.863518: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:31.863518: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:31.863511: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:31.863977: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:31.863986: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:31.863988: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:31.863989: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:31.863993: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:31.863974: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:31.863980: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:31.863979: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:31.867375: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:31.867379: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:31.867378: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:31.867387: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:31.867390: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:31.867379: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:31.867384: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:31.867374: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:31.869702: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:31.869726: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:31.869735: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:31.869736: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:31.869747: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:31.869745: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:31.869743: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:31.869749: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:31.869920: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:31.869936: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:31.869937: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:31.869942: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:31.869922: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:31.869921: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:31.869929: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:31.869948: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:31.870657: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:31.870667: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:31.870663: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:31.870664: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:31.870674: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:31.870676: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:31.870680: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:31.870672: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:31.870876: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:31.870886: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:31.870892: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:31.870894: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:31.870886: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:31.870887: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:31.870900: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:31.870892: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:31.871363: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:31.871379: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:31.871368: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:31.871371: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:31.871379: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:31.871382: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:31.871371: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:31.871380: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:31.871530: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:31.871530: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:31.871537: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:31.871541: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:31.871542: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:31.871550: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:31.871546: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:31.871544: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:31.871770: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:31.871776: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:31.871789: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:31.871787: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:31.871791: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:31.871796: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:31.871796: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:31.871778: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:31.873025: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:31.873028: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:31.873032: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:31.873034: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:31.873033: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:31.873036: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:31.873041: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:31.873023: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:31.873406: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:31.873417: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:31.873410: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:31.873420: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:31.873419: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:31.873425: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:31.873427: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:31.873429: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:31.874385: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:31.874395: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:31.874403: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:31.874413: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:31.874417: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:31.874420: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:31.874426: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:31.874428: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:31.888289: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:31.888309: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:31.888294: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:31.888301: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:31.888292: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:31.888291: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:31.888297: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:31.890105: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:35.672480: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.672522: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.672519: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.672551: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.672549: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.672556: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.672548: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.672582: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.672583: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.672570: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.672585: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.672987: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.672607: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.673020: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:35:35.673021: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.672608: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.673036: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.672616: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.672614: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:35:35.672625: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:35:35.673046: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:35:35.673051: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:35:35.673052: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:35:35.673065: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.673190: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.673219: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.673223: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.673239: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.673242: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.673248: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.673252: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:35:35.673264: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.851460: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.851491: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.851495: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.851514: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.851516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.851529: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.851527: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.851540: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:35:35.851911: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.851928: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.851949: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.851956: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.851951: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.851978: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.851979: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:35:35.851986: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.951367: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.951394: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.951592: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.951407: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.951418: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.951609: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.951421: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.951636: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.951422: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.951643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.951443: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.951630: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.951436: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:35:35.951650: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.951851: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.951659: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.951858: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.951665: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:35:35.951889: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.951892: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.951897: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.951902: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.952009: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.952017: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.952032: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.952033: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.952032: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.952044: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.952057: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.951913: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:35:35.951915: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:35:35.952108: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:36.001814: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:36.001834: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:36.001884: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:36.001846: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:36.001899: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:36.001865: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:36.001912: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:36.001886: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:36.001937: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:36.001876: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:36.001934: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:36.001896: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:36.001930: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:36.001872: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:35:36.001952: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:36.002397: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:36.001941: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:35:36.002422: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:36.002437: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:36.002438: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:36.002437: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:36.002484: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:36.002502: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:36.002459: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:36.002464: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:35:36.002465: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:36.002527: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:36.002527: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:36.002531: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:36.002555: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:36.002563: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:35:36.002571: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:36.080440: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:36.080491: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:36.080494: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:36.080517: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:36.080521: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:36.080534: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:36.080541: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:36.080558: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:35:36.080940: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:36.080974: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:36.080978: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:36.080990: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:36.080991: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:36.081006: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:36.081010: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:35:36.081021: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:36.099487: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:36.101271: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:36.117549: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:36.117458: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:36.100757: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.115716: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:36.099495: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:36.101302: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:36.117085: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:36.116751: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:36.117608: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:36.117579: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:36.117483: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:36.100786: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.115745: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:36.099501: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:36.101300: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:36.117139: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:36.116797: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:36.117603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:36.117632: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:36.117487: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:36.100789: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.115749: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:36.099507: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:36.101317: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:36.117146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:36.116840: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:36.117643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:36.117645: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:36.117504: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:36.100794: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.115792: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:36.099497: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:36.101319: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:36.117167: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:36.116818: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:36.117634: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:36.117665: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:36.117507: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:36.100808: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.115767: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:36.099503: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:36.101326: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:36.117181: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:36.116866: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:36.117649: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:36.117681: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:36.117508: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:36.100803: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.115769: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:36.099499: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:36.101329: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:36.117176: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:36.116858: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:36.117637: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:36.117698: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:36.117519: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:36.100813: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.115767: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:36.099507: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:35:36.101334: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:35:36.117349: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:36.116872: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:36.117655: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:35:36.117710: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:36.117517: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:35:36.100820: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.115790: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:35:36.122474: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:36.122408: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:36.117343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:36.117038: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:35:36.122649: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:36.117702: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:35:36.122620: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:36.122708: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:36.122744: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:35:36.122570: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:36.122479: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:36.122477: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:36.122444: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:36.122657: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:36.122582: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:35:36.122694: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:35:36.122700: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:36.122682: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:36.122762: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.122462: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.122621: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:36.122459: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:36.122467: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:36.122612: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:36.122726: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:36.122733: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:36.122680: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:36.122692: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:36.122787: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:36.122797: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:36.122786: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.122478: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.122629: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:36.122528: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:36.122536: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:36.122489: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:36.122706: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:36.122710: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:36.122640: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:35:36.122724: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:35:36.122725: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:36.122725: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:36.122723: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:36.122724: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:36.122736: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:36.122813: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.122526: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.122646: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:36.122552: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:36.122696: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:36.122698: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:35:36.122716: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:36.122726: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:36.122733: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:35:36.122734: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:36.122745: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:35:36.122818: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:35:36.122915: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.122487: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.122652: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:36.122645: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:35:36.122647: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:36.122909: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:36.122911: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:35:36.122927: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:35:36.122742: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:36.122757: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.122662: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:36.122658: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:36.122656: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:35:36.122870: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:36.122784: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:36.122796: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.122491: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:36.122668: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:35:36.122804: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.122525: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:35:36.122789: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:35:36.122791: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:35:36.122817: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:35:36.122703: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.122668: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:35:36.122899: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.123006: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.123057: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.123070: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.123073: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.123064: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.123093: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.123111: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:35:36.123134: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:36.174584: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:36.174612: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:36.174618: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:36.174639: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:36.174638: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:36.174655: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:36.174652: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:36.174656: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:35:36.175081: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:36.175112: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:36.175117: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:36.175136: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:36.175141: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:36.175142: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:36.175157: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:35:36.175195: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:36.436612: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:36.436644: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:36.436646: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:36.436665: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:36.436667: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:36.436677: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:36.436691: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:36.436674: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:35:36.437002: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:36.437008: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:36.437037: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:36.437042: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:36.437054: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:36.437049: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:36.437065: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:35:36.437071: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:36.483242: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:36.483261: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:36.483270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:36.483289: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:36.483296: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:36.483303: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:36.483310: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:36.483313: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:35:36.483977: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:36.483997: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:36.484009: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:36.484028: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:36.484035: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:36.484042: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:36.484055: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:35:36.484054: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:36.492782: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:36.492783: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:36.492788: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:36.492795: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:36.492792: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:36.492791: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:36.492796: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:36.492802: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:35:36.493174: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:36.493178: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:36.493182: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:36.493183: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:36.493188: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:36.493186: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:36.493192: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:35:36.493196: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:36.497678: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:36.497707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:36.497709: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:36.497759: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:36.497752: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:36.497748: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:36.497743: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:36.497732: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:35:36.498174: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:36.498184: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:36.498192: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:36.498187: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:36.498220: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:36.498228: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:36.498228: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:35:36.498238: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:36.498740: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:36.498768: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:36.498772: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:36.498788: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:36.498787: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:36.498791: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:36.498796: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:36.498803: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:35:36.499359: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:36.499388: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:36.499394: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:36.499408: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:36.499412: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:36.499414: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:36.499422: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:35:36.499424: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:36.571956: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:36.571964: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:36.571960: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:36.571967: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:36.571967: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:36.571971: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:36.571968: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:36.571973: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:35:36.572327: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:36.572331: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:36.572336: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:36.572339: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:36.572341: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:36.572341: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:36.572346: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:35:36.572348: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:36.574156: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:36.574174: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:36.574191: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:36.574193: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:36.574208: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:36.574208: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:36.574199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:36.574224: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:35:36.574557: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:36.574586: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:36.574589: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:36.574607: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:36.574615: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:36.574614: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:36.574623: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:35:36.574638: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:36.576200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:36.576261: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:36.576228: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:36.576207: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:36.576217: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:36.576289: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:36.576246: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:36.576261: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:36.576308: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:36.576252: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:36.576282: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:36.576292: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:36.576261: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:36.576256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:36.576318: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:36.576249: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:36.576277: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:36.576302: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:36.576243: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:36.576295: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:36.576310: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:36.576297: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:36.576329: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:36.576315: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:36.576623: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:36.576654: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:36.577144: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:36.577150: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:36.577137: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:36.577143: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:36.576657: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:35:36.576829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:36.577167: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:36.577161: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:36.577168: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:36.577169: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:36.577172: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:36.576683: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:36.576856: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:36.577114: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:36.577177: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.576858: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:36.577189: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:36.577191: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:36.577189: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:36.576688: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:36.576884: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:36.577207: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:36.577204: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:36.577211: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.576859: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:35:36.577201: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:35:36.577207: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:35:36.577190: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:36.576693: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:36.576893: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:36.577225: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.576882: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:35:36.577200: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:36.576703: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:36.576891: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:35:36.577235: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:36.577241: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:35:36.577245: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.576888: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:36.576699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:36.576910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:36.577144: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:36.577149: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.576890: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:36.577158: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:36.576910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:35:36.576880: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:36.577187: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:36.577193: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:36.576921: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:35:36.576898: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:35:36.577213: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:36.577217: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:36.576919: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:35:36.577167: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:36.577169: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.577244: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:36.577231: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:36.577233: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:35:36.577237: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:36.577172: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.577272: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.577277: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:36.577182: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:35:36.577206: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.577300: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.577302: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.577308: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.577309: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:35:36.577315: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:36:08.396919: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.396961: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.396982: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397006: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397020: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397026: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397046: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397062: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397941: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397947: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397957: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.397949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.397939: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.397967: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.397989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397953: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.397995: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.398025: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.398082: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397958: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397973: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.398028: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.398242: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.398200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397975: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.397976: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.397977: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.398030: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.398113: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.398293: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:08.397977: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.397979: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:08.397980: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.398040: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.398278: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.398124: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.398231: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.398236: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.398289: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.398146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.398327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.398287: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.398317: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.398163: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.398323: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.398296: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.398328: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.398169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.398344: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.398308: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.398352: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.398256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.398353: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.398176: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.398346: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.398303: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.398290: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.398351: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.398191: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.398352: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.398323: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.398391: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.398302: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.398360: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.398328: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.398400: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.398323: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.398371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.398421: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.398315: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.398426: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.398332: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.398344: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.398592: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.398596: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.398613: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398541: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.398549: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.398585: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.398607: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398572: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.398630: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.398578: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398582: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.398642: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.398595: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.398648: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.398623: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398612: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.398643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.398649: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398629: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.398664: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.398651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398631: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.398673: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.398658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.398642: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.398706: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.398978: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.399025: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.398974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.399063: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.399077: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.399094: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.399128: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.399122: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.399123: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.399032: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.399007: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.399025: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.399039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.399053: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.399155: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.399157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.399175: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.399059: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.399081: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.399087: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.399103: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.399103: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.399127: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.399131: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.401160: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.401182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.401189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.401199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.401206: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.401211: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.401219: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.401220: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.401479: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.401480: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.401482: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.401496: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.401496: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.401496: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.401500: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.401501: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.401502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.401501: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.401502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:08.401521: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.401522: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.401523: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.401523: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:08.401524: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.401733: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.401761: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.401751: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.401764: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.401795: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401844: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.401758: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.401766: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.401802: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401847: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.401766: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.401767: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.401803: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.401773: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.401772: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.401801: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401855: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.401778: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.401774: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.401806: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401855: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401865: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.401783: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.401774: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.401780: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.401807: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401862: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.401906: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.401777: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.401780: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.401808: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.401815: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401857: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.402029: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402050: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.401781: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:08.401785: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:08.401788: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.401817: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.401820: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.401820: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401859: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:08.401866: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.402039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:08.401790: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:08.401792: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:08.401794: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.402031: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.401816: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.401823: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401877: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401879: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401880: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.402041: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402051: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:08.401824: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.401825: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:08.401834: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401877: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401881: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:08.401882: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.402039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402051: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.402033: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.402213: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.402041: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402055: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.402032: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.402047: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402055: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.402034: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.402218: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.402232: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.402043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.402055: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.402037: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.402233: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.402226: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.402045: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.402065: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402055: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.402038: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.402229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.402068: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.402069: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.402069: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402069: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.402039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.402231: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:08.402071: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.402071: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:08.402070: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402069: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402070: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402072: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.402043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:08.402064: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.402230: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402074: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402075: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:08.402076: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.402071: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.402073: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.402073: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.402230: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:08.402076: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.402075: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.402076: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.402079: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.402237: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:08.402255: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:08.402077: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.402255: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.402255: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.402257: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.402257: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:08.402258: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.402320: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.402347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402311: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.402330: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.402246: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.402327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.402332: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402325: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.402251: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.402329: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402346: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.402257: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.402334: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.402386: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402351: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.402262: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.402328: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.402421: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402360: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.402260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.402338: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.402357: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.402459: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.402303: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.402266: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:08.402366: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.402373: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.402373: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.402485: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402384: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.402264: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.402318: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.402374: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.402376: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.402378: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.402491: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.402597: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402565: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.402319: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:08.402318: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:08.402318: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:08.402378: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:08.402567: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.402324: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.402323: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:08.402321: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:08.402321: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:08.402567: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.402327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.402403: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:08.402453: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:08.402574: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.402340: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402573: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.402351: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.402686: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402570: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.402364: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402573: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.402508: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.402658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402574: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402590: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.402516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402590: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:08.402590: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:08.402590: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:08.402597: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:08.402594: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:08.402599: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.402658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:08.402600: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.402667: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.402668: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.402672: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.402664: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.402664: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.402682: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.402682: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.402682: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:08.402689: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.402691: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.402690: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.402690: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.402688: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:08.402702: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.402684: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.403220: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.403222: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.403224: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.403224: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.403189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.403222: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.403226: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.403230: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.403236: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.403236: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.403233: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.403229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.403245: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.403245: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.403246: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.403245: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:08.403246: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.403249: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:08.403254: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.403276: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.403287: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.403298: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.403327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.403496: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.403564: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.403580: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.403575: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.403638: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.403578: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.403579: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.403577: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.403583: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.403582: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.403584: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:08.403601: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.403602: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.403603: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.403606: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.403604: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.403605: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:08.403606: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.404360: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.404365: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.404366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.404369: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.404372: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.404368: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.404371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.404380: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.404381: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.404385: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.404388: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.404388: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.404390: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.404378: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:08.404391: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:08.404403: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.404651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.404660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.404666: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.404674: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.404680: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.404687: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.404689: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.404691: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.404691: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.404694: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:08.404703: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.404706: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.404708: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.404710: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.404710: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:08.404711: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:08.405810: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.405811: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.405813: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.405819: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.405824: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.405824: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.405837: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:08.405838: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:08.405837: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:08.405846: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:08.405847: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:08.405849: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:08.405981: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.406008: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.408217: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.408246: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.408264: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.408272: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.408282: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.408280: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.408289: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.408443: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.409498: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.409500: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.409502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.409505: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.409507: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.409509: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.409505: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.409504: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:08.409515: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.409515: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.409518: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.409523: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.409524: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.409524: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.409525: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:08.409527: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.410202: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.410230: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.410250: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.410266: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.410285: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.410291: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.410296: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.410491: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.410919: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.410956: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.410979: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.410987: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.410998: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.411017: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.411239: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.411278: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.411614: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.411620: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.411621: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.411620: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.411622: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.411623: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.411630: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.411625: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.411634: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.411627: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:08.411637: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.411641: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.411644: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.411642: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.411642: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:08.411646: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.412286: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.423536: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.423569: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.423583: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.423608: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.423626: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.423636: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.423642: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.423869: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.424893: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.424917: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.424911: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.424916: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.424921: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.424912: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.424923: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.424913: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.424921: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:08.424959: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.424961: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.424961: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.424964: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.424965: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.424966: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:08.424967: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.433030: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.433062: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.433077: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.433094: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.433102: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.433105: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.433118: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.433307: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.434599: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.434600: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.434600: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.434603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.434602: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.434606: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.434606: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.434606: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:08.434621: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.434621: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.434625: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.434624: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.434627: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.434621: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.434626: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:08.434627: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.398376: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.401973: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.401977: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.401989: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.401978: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.401989: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.401978: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.401982: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.401982: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.402013: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.402015: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.402015: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.402016: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.402124: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.402141: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:08.402141: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:08.402155: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.442232: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442269: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442283: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442303: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442332: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442337: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442355: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.442500: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.443422: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.443428: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.443431: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.443433: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.443433: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.443446: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.443436: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.443451: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.443439: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.443425: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.443438: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.443453: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.443457: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.443458: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.443460: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.443452: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:08.443466: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:08.443467: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.443496: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.443493: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.443512: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.443523: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.443536: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.443738: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.443726: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.443769: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.443809: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.443810: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.443828: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.443844: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.443854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.444049: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.444626: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.444688: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.444651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.444676: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.444689: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.444694: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.444701: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.444699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.444706: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.444711: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.444705: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.444699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.444726: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.444701: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.444703: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.444707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.444703: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.444723: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.444916: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:08.444725: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.444726: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.444727: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.444727: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:08.444728: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.403655: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403650: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.403651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.403653: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.403658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.403668: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403659: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.403664: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:08.403675: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403676: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403677: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403684: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403687: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:08.403688: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.445210: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445231: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.445088: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445230: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445231: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445234: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445234: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.445130: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445241: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.445149: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445243: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445257: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.445169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445259: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.445259: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.445261: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.445196: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445263: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:08.445264: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.445202: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445297: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:08.445317: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.445208: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.445396: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.446006: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.446017: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.446015: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.446024: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.445883: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.446018: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.446018: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.446018: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.445921: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.446032: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.446023: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.445936: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.446023: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.446039: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.445950: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.446040: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.446041: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.446043: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.445978: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:08.446044: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:08.446045: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.445979: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.446043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.446067: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.446460: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.446475: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.446469: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.446371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.446474: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.446474: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.446477: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.446479: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.446400: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.446482: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.446431: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.446482: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.446496: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.446428: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.446499: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.446499: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.446501: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.446455: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:08.446502: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.446504: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:08.446504: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.446458: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.446660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.446661: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.447415: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.447418: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.447416: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.447430: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.447420: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.447422: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.447421: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.447421: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.447423: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:08.447441: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.447444: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.447445: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.447444: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.447446: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.447446: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:08.447443: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.447665: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.406002: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:08.406030: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.412289: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.412291: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.412292: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.412291: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.412293: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.412303: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.412312: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.412313: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.412312: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.412313: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.412314: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.412314: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.412320: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:08.412318: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:08.412347: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.447669: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.447672: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.447673: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.447675: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.447685: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.447677: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.447687: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.447677: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.447690: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.447693: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.447694: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.447681: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:08.447699: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.447704: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:08.447707: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: Loading extension module scaled_upper_triang_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module scaled_masked_softmax_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module scaled_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module fused_mix_prec_layer_norm_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module fused_mix_prec_layer_norm_cuda... +31: Successfully preprocessed all matching files. + 2: Successfully preprocessed all matching files. + 4: Successfully preprocessed all matching files. +14: Successfully preprocessed all matching files. + 5: Successfully preprocessed all matching files. + 5: Successfully preprocessed all matching files. + 0: Successfully preprocessed all matching files. + 0: Successfully preprocessed all matching files. + 7: Successfully preprocessed all matching files. + 7: Successfully preprocessed all matching files. + 0: Successfully preprocessed all matching files. +13: Successfully preprocessed all matching files. +15: Successfully preprocessed all matching files. +15: Successfully preprocessed all matching files. + 8: Successfully preprocessed all matching files. +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: + 1: + 1: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: + 3: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: + 6: + 6: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +10: +10: + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: + 9: + 9: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: +11: +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +12: +12: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +18: +18: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +17: +17: +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: +16: +16: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +20: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +19: +19: +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +21: +21: +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: +24: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +22: +22: +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: +23: +23: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: +26: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: +25: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: +27: +27: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +28: +28: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: +29: +29: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: +30: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +14: Building extension module utils... +14: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +14: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... + 0: Building extension module utils... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... +27: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 2: Loading extension module utils... + 0: Loading extension module utils... + 2: Loading extension module utils... + 0: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 4: Loading extension module utils... + 2: Loading extension module utils... + 4: Loading extension module utils... + 5: Loading extension module utils... + 5: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... + 5: Loading extension module utils... + 4: Loading extension module utils... + 8: Loading extension module utils... + 5: Loading extension module utils... + 4: Loading extension module utils... + 8: Loading extension module utils... + 7: Loading extension module utils... + 5: Loading extension module utils... + 1: Loading extension module utils... + 4: Loading extension module utils... + 1: Loading extension module utils... + 5: Loading extension module utils... + 4: Loading extension module utils... + 7: Loading extension module utils... + 8: Loading extension module utils... + 1: Loading extension module utils... + 5: Loading extension module utils... + 1: Loading extension module utils... + 7: Loading extension module utils... + 8: Loading extension module utils... + 1: Loading extension module utils... + 7: Loading extension module utils... + 8: Loading extension module utils... + 5: Loading extension module utils... + 1: Loading extension module utils... + 7: Loading extension module utils... + 3: Loading extension module utils... + 8: Loading extension module utils... + 1: Loading extension module utils... + 7: Loading extension module utils... + 3: Loading extension module utils... + 7: Loading extension module utils... + 8: Loading extension module utils... + 1: Loading extension module utils... + 3: Loading extension module utils... + 8: Loading extension module utils... + 7: Loading extension module utils... + 3: Loading extension module utils... + 3: Loading extension module utils... + 3: Loading extension module utils... + 3: Loading extension module utils... + 3: Loading extension module utils... + 6: Loading extension module utils... + 6: Loading extension module utils... + 6: Loading extension module utils... + 6: Loading extension module utils... +15: Loading extension module utils... +15: Loading extension module utils... +15: Loading extension module utils... + 9: Loading extension module utils... +10: Loading extension module utils... +15: Loading extension module utils... +10: Loading extension module utils... +10: Loading extension module utils... + 9: Loading extension module utils... + 9: Loading extension module utils... +10: Loading extension module utils... + 9: Loading extension module utils... +15: Loading extension module utils... +15: Loading extension module utils... +13: Loading extension module utils... +15: Loading extension module utils... + 6: Loading extension module utils... +15: Loading extension module utils... +13: Loading extension module utils... + 6: Loading extension module utils... +11: Loading extension module utils... +13: Loading extension module utils... + 6: Loading extension module utils... + 6: Loading extension module utils... +11: Loading extension module utils... +10: Loading extension module utils... +12: Loading extension module utils... +11: Loading extension module utils... +10: Loading extension module utils... +13: Loading extension module utils... +10: Loading extension module utils... +13: Loading extension module utils... +10: Loading extension module utils... +12: Loading extension module utils... +11: Loading extension module utils... +13: Loading extension module utils... +12: Loading extension module utils... +13: Loading extension module utils... +12: Loading extension module utils... +13: Loading extension module utils... +17: Loading extension module utils... +17: Loading extension module utils... + 9: Loading extension module utils... +16: Loading extension module utils... + 9: Loading extension module utils... +17: Loading extension module utils... +11: Loading extension module utils... +12: Loading extension module utils... +16: Loading extension module utils... + 9: Loading extension module utils... +11: Loading extension module utils... +12: Loading extension module utils... +17: Loading extension module utils... +11: Loading extension module utils... +16: Loading extension module utils... + 9: Loading extension module utils... +12: Loading extension module utils... +11: Loading extension module utils... +16: Loading extension module utils... +12: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +17: Loading extension module utils... +16: Loading extension module utils... +18: Loading extension module utils... +17: Loading extension module utils... +16: Loading extension module utils... +18: Loading extension module utils... +17: Loading extension module utils... +16: Loading extension module utils... +18: Loading extension module utils... +16: Loading extension module utils... +17: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +20: Loading extension module utils... +20: Loading extension module utils... +21: Loading extension module utils... +20: Loading extension module utils... +19: Loading extension module utils... +20: Loading extension module utils... +14: Loading extension module utils... +21: Loading extension module utils... +19: Loading extension module utils... +21: Loading extension module utils... +20: Loading extension module utils... +21: Loading extension module utils... +19: Loading extension module utils... +20: Loading extension module utils... +22: Loading extension module utils... +24: Loading extension module utils... +20: Loading extension module utils... +19: Loading extension module utils... +20: Loading extension module utils... +21: Loading extension module utils... +22: Loading extension module utils... +24: Loading extension module utils... +19: Loading extension module utils... +21: Loading extension module utils... +19: Loading extension module utils... +21: Loading extension module utils... +19: Loading extension module utils... +22: Loading extension module utils... +21: Loading extension module utils... +19: Loading extension module utils... +22: Loading extension module utils... +24: Loading extension module utils... +23: Loading extension module utils... +26: Loading extension module utils... +24: Loading extension module utils... +24: Loading extension module utils... +26: Loading extension module utils... +23: Loading extension module utils... +22: Loading extension module utils... +24: Loading extension module utils... +25: Loading extension module utils... +24: Loading extension module utils... +26: Loading extension module utils... +23: Loading extension module utils... +22: Loading extension module utils... +24: Loading extension module utils... +26: Loading extension module utils... +22: Loading extension module utils... +23: Loading extension module utils... +25: Loading extension module utils... +22: Loading extension module utils... +23: Loading extension module utils... +26: Loading extension module utils... +26: Loading extension module utils... +23: Loading extension module utils... +26: Loading extension module utils... +23: Loading extension module utils... +26: Loading extension module utils... +29: Loading extension module utils... +23: Loading extension module utils... +25: Loading extension module utils... +28: Loading extension module utils... +27: Loading extension module utils... +25: Loading extension module utils... +25: Loading extension module utils... +27: Loading extension module utils... +25: Loading extension module utils... +29: Loading extension module utils... +27: Loading extension module utils... +25: Loading extension module utils... +31: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +29: Loading extension module utils... +25: Loading extension module utils... +29: Loading extension module utils... +31: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +29: Loading extension module utils... +28: Loading extension module utils... +29: Loading extension module utils... +31: Loading extension module utils... +27: Loading extension module utils... +28: Loading extension module utils... +29: Loading extension module utils... +27: Loading extension module utils... +31: Loading extension module utils... +28: Loading extension module utils... +29: Loading extension module utils... +27: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +27: Loading extension module utils... +30: Loading extension module utils... +31: Loading extension module utils... +14: Loading extension module utils... +31: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Loading extension module utils... + 0: + 0: No modifications detected for re-loaded extension module utils, skipping build step...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: + 0: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... +14: Loading extension module utils... +14: Loading extension module utils... +14: Loading extension module utils... +14: Loading extension module utils... +14: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: +27: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: +27: Loading extension module utils... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: +27: Loading extension module utils... +27: Loading extension module utils... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: + 8: + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 8: + 8: Loading extension module utils...Loading extension module utils... + 8: + 8: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 8: + 8: Loading extension module utils...Loading extension module utils... + 8: + 8: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 8: + 8: + 8: Loading extension module utils...Loading extension module utils...Loading extension module utils... + 8: + 8: + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 7: + 7: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 7: + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: + 5: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 5: + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 5: + 5: Loading extension module utils...Loading extension module utils... + 5: + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +15: +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: +21: Loading extension module utils... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +16: +16: +16: Loading extension module utils...Loading extension module utils... +16: +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +16: +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +21: +21: +21: Loading extension module utils...Loading extension module utils...Loading extension module utils... +21: +21: +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 3: + 3: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +11: +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +28: +28: Loading extension module utils...Loading extension module utils... +28: +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 6: + 6: Loading extension module utils... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +23: +23: Loading extension module utils...Loading extension module utils... +23: +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +23: +23: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +23: +23: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +19: +19: Loading extension module utils...Loading extension module utils... +19: +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +19: +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +22: +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: +29: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +29: +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils...Loading extension module utils... +29: +29: +29: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: +29: +29: Loading extension module utils... +29: Loading extension module utils... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: +13: +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +10: +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +13: +13: Loading extension module utils...Loading extension module utils... +13: +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +13: +13: +13: Loading extension module utils...Loading extension module utils...Loading extension module utils... +13: +13: +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: +10: Loading extension module utils...Loading extension module utils... +10: +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +26: +26: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +26: +26: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: Loading extension module utils...Loading extension module utils... +20: +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: Loading extension module utils...Loading extension module utils... +20: +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: Loading extension module utils...Loading extension module utils... +20: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +18: +18: Loading extension module utils...Loading extension module utils... +18: +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +24: +24: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... +24: +24: +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +25: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +25: +25: Loading extension module utils...Loading extension module utils... +25: +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +30: +30: Loading extension module utils... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +30: +30: +30: +30: Loading extension module utils...Loading extension module utils...Loading extension module utils...Loading extension module utils... +30: +30: +30: +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: + 1: Loading extension module utils... + 1: Loading extension module utils... + 1: + 1: Loading extension module utils... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +17: +17: Loading extension module utils...Loading extension module utils... +17: +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 9: + 9: Loading extension module utils...Loading extension module utils... + 9: + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +12: +12: Loading extension module utils...Loading extension module utils... +12: + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 9: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +17: +17: Loading extension module utils... +17: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 9: + 9: Loading extension module utils...Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +12: +12: Loading extension module utils... + 9: + 9: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 2: + 2: Loading extension module utils...Loading extension module utils... + 2: +12: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +12: +12: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... +12: +12: +12: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 2: + 2: Loading extension module utils... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings + 0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/perplexity25/3510898.out b/perplexity25/3510898.out new file mode 100644 index 0000000000000000000000000000000000000000..049f2bf991f1a8f582003a475aea986ebc9a1e02 --- /dev/null +++ b/perplexity25/3510898.out @@ -0,0 +1,42549 @@ +Model parameters: d_model 3072 ffw_size 12288 kv_size 128 n_heads 24 n_layers 36 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 2 --pipeline-model-parallel-size 1 --num-layers 36 --hidden-size 3072 --num-attention-heads 24 --kv-channels 128 --ffn-hidden-size 12288 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 2 --global-batch-size 1024 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-4b284bc4perplexity25val --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --override-lr-scheduler --reset-progress --no-load-optim --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --eval-only true --tensorboard-dir tensorboard_4b284bc4perplexity25val --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_4b284bc4perplexity25 --load checkpoints_4b284bc4perplexity25 --train-weighted-split-paths-path train1b5.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/3510898.json --zero-stage 0 +START 3510898: Sat 13 May 2023 09:34:45 AM EEST + 0: + 0: + 0: ======================= ROCm System Management Interface ======================= + 0: ================================= Concise Info ================================= + 0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 0: 0 45.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 2 45.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 3 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 4 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 6 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 7 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: ================================================================================ + 0: ============================= End of ROCm SMI Log ============================== + 4: + 4: + 4: ======================= ROCm System Management Interface ======================= + 4: ================================= Concise Info ================================= + 4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 4: 0 52.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 2 46.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 4 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 6 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: ================================================================================ + 4: ============================= End of ROCm SMI Log ============================== + 3: + 3: + 3: ======================= ROCm System Management Interface ======================= + 3: ================================= Concise Info ================================= + 3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 3: 0 51.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 2 40.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 4 45.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 6 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: ================================================================================ + 3: ============================= End of ROCm SMI Log ============================== +21: +21: +21: ======================= ROCm System Management Interface ======================= +21: ================================= Concise Info ================================= +21: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +21: 0 50.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 2 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 4 47.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 6 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: ================================================================================ +21: ============================= End of ROCm SMI Log ============================== +12: +12: +12: ======================= ROCm System Management Interface ======================= +12: ================================= Concise Info ================================= +12: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +12: 0 48.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 2 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 4 47.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 6 43.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: ================================================================================ +12: ============================= End of ROCm SMI Log ============================== +18: +18: +18: ======================= ROCm System Management Interface ======================= +18: ================================= Concise Info ================================= +18: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +18: 0 48.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 1 53.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 2 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 4 49.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 6 39.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: ================================================================================ +18: ============================= End of ROCm SMI Log ============================== +25: +25: +25: ======================= ROCm System Management Interface ======================= +25: ================================= Concise Info ================================= +25: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +25: 0 46.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 2 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 4 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 6 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: ================================================================================ +25: ============================= End of ROCm SMI Log ============================== +24: +24: +24: ======================= ROCm System Management Interface ======================= +24: ================================= Concise Info ================================= +24: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +24: 0 45.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 2 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 4 42.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 6 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: ================================================================================ +24: ============================= End of ROCm SMI Log ============================== +11: +11: +11: ======================= ROCm System Management Interface ======================= +11: ================================= Concise Info ================================= +11: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +11: 0 48.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 2 45.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 4 49.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 6 45.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: ================================================================================ +11: ============================= End of ROCm SMI Log ============================== + 7: + 7: + 7: ======================= ROCm System Management Interface ======================= + 7: ================================= Concise Info ================================= + 7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 7: 0 46.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 2 43.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 4 46.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 6 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: ================================================================================ + 7: ============================= End of ROCm SMI Log ============================== +26: +26: +26: ======================= ROCm System Management Interface ======================= +26: ================================= Concise Info ================================= +26: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +26: 0 49.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 1 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 2 42.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 4 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 5 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 6 42.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: ================================================================================ +26: ============================= End of ROCm SMI Log ============================== +31: +31: +31: ======================= ROCm System Management Interface ======================= +31: ================================= Concise Info ================================= +31: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +31: 0 48.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 2 42.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 4 45.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 6 40.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: ================================================================================ +31: ============================= End of ROCm SMI Log ============================== +23: +23: +23: ======================= ROCm System Management Interface ======================= +23: ================================= Concise Info ================================= +23: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +23: 0 43.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 2 44.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 4 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 6 40.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: ================================================================================ +23: ============================= End of ROCm SMI Log ============================== +28: +28: +28: ======================= ROCm System Management Interface ======================= +28: ================================= Concise Info ================================= +28: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +28: 0 44.0c 100.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 2 42.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 4 40.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 6 42.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: ================================================================================ +28: ============================= End of ROCm SMI Log ============================== +17: +17: +17: ======================= ROCm System Management Interface ======================= +17: ================================= Concise Info ================================= +17: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +17: 0 50.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 2 44.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 4 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 6 43.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: ================================================================================ +17: ============================= End of ROCm SMI Log ============================== +14: +14: +14: ======================= ROCm System Management Interface ======================= +14: ================================= Concise Info ================================= +14: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +14: 0 46.0c 99.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 2 38.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 4 50.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 6 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: ================================================================================ +14: ============================= End of ROCm SMI Log ============================== + 8: + 8: + 8: ======================= ROCm System Management Interface ======================= + 8: ================================= Concise Info ================================= + 8: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 8: 0 48.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 1 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 2 40.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 4 45.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 6 45.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: ================================================================================ + 8: ============================= End of ROCm SMI Log ============================== + 5: + 5: + 5: ======================= ROCm System Management Interface ======================= + 5: ================================= Concise Info ================================= + 5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 5: 0 42.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 1 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 2 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 3 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 4 46.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 6 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: ================================================================================ + 5: ============================= End of ROCm SMI Log ============================== +13: +13: +13: ======================= ROCm System Management Interface ======================= +13: ================================= Concise Info ================================= +13: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +13: 0 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 2 43.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 4 43.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 6 38.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 7 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: ================================================================================ +13: ============================= End of ROCm SMI Log ============================== +16: +16: +16: ======================= ROCm System Management Interface ======================= +16: ================================= Concise Info ================================= +16: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +16: 0 44.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 2 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 4 43.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 6 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: ================================================================================ +16: ============================= End of ROCm SMI Log ============================== +19: +19: +19: ======================= ROCm System Management Interface ======================= +19: ================================= Concise Info ================================= +19: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +19: 0 45.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 2 48.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 4 46.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 6 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: ================================================================================ +19: ============================= End of ROCm SMI Log ============================== + 6: + 6: + 6: ======================= ROCm System Management Interface ======================= + 6: ================================= Concise Info ================================= + 6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 6: 0 48.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 2 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 4 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 6 45.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 7 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: ================================================================================ + 6: ============================= End of ROCm SMI Log ============================== +15: +15: +15: ======================= ROCm System Management Interface ======================= +15: ================================= Concise Info ================================= +15: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +15: 0 47.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 2 39.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 4 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 6 37.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: ================================================================================ +15: ============================= End of ROCm SMI Log ============================== +20: +20: +20: ======================= ROCm System Management Interface ======================= +20: ================================= Concise Info ================================= +20: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +20: 0 51.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 2 46.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 4 46.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 6 38.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 7 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: ================================================================================ +20: ============================= End of ROCm SMI Log ============================== +10: +10: +10: ======================= ROCm System Management Interface ======================= +10: ================================= Concise Info ================================= +10: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +10: 0 45.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 2 37.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 4 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 6 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 7 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: ================================================================================ +10: ============================= End of ROCm SMI Log ============================== +29: +29: +29: ======================= ROCm System Management Interface ======================= +29: ================================= Concise Info ================================= +29: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +29: 0 48.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 2 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 4 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 6 40.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: ================================================================================ +29: ============================= End of ROCm SMI Log ============================== + 2: + 2: + 2: ======================= ROCm System Management Interface ======================= + 2: ================================= Concise Info ================================= + 2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 2: 0 49.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 2 44.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 4 54.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 6 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 7 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: ================================================================================ + 2: ============================= End of ROCm SMI Log ============================== +27: +27: +27: ======================= ROCm System Management Interface ======================= +27: ================================= Concise Info ================================= +27: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +27: 0 45.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 2 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 4 45.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 6 40.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: ================================================================================ +27: ============================= End of ROCm SMI Log ============================== + 9: + 9: + 9: ======================= ROCm System Management Interface ======================= + 9: ================================= Concise Info ================================= + 9: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 9: 0 47.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 2 51.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 4 37.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 6 47.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: ================================================================================ + 9: ============================= End of ROCm SMI Log ============================== +22: +22: +22: ======================= ROCm System Management Interface ======================= +22: ================================= Concise Info ================================= +22: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +22: 0 45.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 2 40.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 4 39.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 6 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: ================================================================================ +22: ============================= End of ROCm SMI Log ============================== + 1: + 1: + 1: ======================= ROCm System Management Interface ======================= + 1: ================================= Concise Info ================================= + 1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 1: 0 50.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 2 44.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 4 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 5 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 6 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: ================================================================================ + 1: ============================= End of ROCm SMI Log ============================== +30: +30: +30: ======================= ROCm System Management Interface ======================= +30: ================================= Concise Info ================================= +30: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +30: 0 46.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 2 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 4 40.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 5 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 6 40.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: ================================================================================ +30: ============================= End of ROCm SMI Log ============================== +17: Launching on nid006792 (17/32), master nid006775 port 9999, GPUs 8, CUDA: True +18: Launching on nid006793 (18/32), master nid006775 port 9999, GPUs 8, CUDA: True +26: Launching on nid006984 (26/32), master nid006775 port 9999, GPUs 8, CUDA: True +21: Launching on nid006796 (21/32), master nid006775 port 9999, GPUs 8, CUDA: True +31: Launching on nid006989 (31/32), master nid006775 port 9999, GPUs 8, CUDA: True +24: Launching on nid006982 (24/32), master nid006775 port 9999, GPUs 8, CUDA: True + 4: Launching on nid006779 (4/32), master nid006775 port 9999, GPUs 8, CUDA: True + 7: Launching on nid006782 (7/32), master nid006775 port 9999, GPUs 8, CUDA: True +28: Launching on nid006986 (28/32), master nid006775 port 9999, GPUs 8, CUDA: True +14: Launching on nid006789 (14/32), master nid006775 port 9999, GPUs 8, CUDA: True + 3: Launching on nid006778 (3/32), master nid006775 port 9999, GPUs 8, CUDA: True +11: Launching on nid006786 (11/32), master nid006775 port 9999, GPUs 8, CUDA: True + 0: Launching on nid006775 (0/32), master nid006775 port 9999, GPUs 8, CUDA: True +12: Launching on nid006787 (12/32), master nid006775 port 9999, GPUs 8, CUDA: True +13: Launching on nid006788 (13/32), master nid006775 port 9999, GPUs 8, CUDA: True +25: Launching on nid006983 (25/32), master nid006775 port 9999, GPUs 8, CUDA: True + 8: Launching on nid006783 (8/32), master nid006775 port 9999, GPUs 8, CUDA: True +23: Launching on nid006981 (23/32), master nid006775 port 9999, GPUs 8, CUDA: True + 5: Launching on nid006780 (5/32), master nid006775 port 9999, GPUs 8, CUDA: True +16: Launching on nid006791 (16/32), master nid006775 port 9999, GPUs 8, CUDA: True +19: Launching on nid006794 (19/32), master nid006775 port 9999, GPUs 8, CUDA: True +10: Launching on nid006785 (10/32), master nid006775 port 9999, GPUs 8, CUDA: True + 6: Launching on nid006781 (6/32), master nid006775 port 9999, GPUs 8, CUDA: True +27: Launching on nid006985 (27/32), master nid006775 port 9999, GPUs 8, CUDA: True +20: Launching on nid006795 (20/32), master nid006775 port 9999, GPUs 8, CUDA: True +30: Launching on nid006988 (30/32), master nid006775 port 9999, GPUs 8, CUDA: True +29: Launching on nid006987 (29/32), master nid006775 port 9999, GPUs 8, CUDA: True +15: Launching on nid006790 (15/32), master nid006775 port 9999, GPUs 8, CUDA: True + 2: Launching on nid006777 (2/32), master nid006775 port 9999, GPUs 8, CUDA: True + 9: Launching on nid006784 (9/32), master nid006775 port 9999, GPUs 8, CUDA: True + 1: Launching on nid006776 (1/32), master nid006775 port 9999, GPUs 8, CUDA: True +22: Launching on nid006797 (22/32), master nid006775 port 9999, GPUs 8, CUDA: True + 0: using world size: 256, data-parallel-size: 128, tensor-model-parallel size: 2, pipeline-model-parallel size: 1 + 0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. + 0: using torch.bfloat16 for parameters ... + 0: ------------------------ arguments ------------------------ + 0: abort_on_unmet_fused_kernel_constraints ......... False + 0: accumulate_allreduce_grads_in_fp32 .............. True + 0: adam_beta1 ...................................... 0.9 + 0: adam_beta2 ...................................... 0.999 + 0: adam_eps ........................................ 1e-08 + 0: adlr_autoresume ................................. False + 0: adlr_autoresume_interval ........................ 1000 + 0: apply_query_key_layer_scaling ................... True + 0: apply_residual_connection_post_layernorm ........ False + 0: attention_dropout ............................... 0.1 + 0: attention_softmax_in_fp32 ....................... False + 0: bert_binary_head ................................ True + 0: bert_load ....................................... None + 0: bf16 ............................................ True + 0: bias_dropout_fusion ............................. True + 0: bias_gelu_fusion ................................ True + 0: biencoder_projection_dim ........................ 0 + 0: biencoder_shared_query_context_model ............ False + 0: block_data_path ................................. None + 0: checkpoint_activations .......................... False + 0: checkpoint_in_cpu ............................... False + 0: checkpoint_num_layers ........................... 1 + 0: clip_grad ....................................... 1.0 + 0: codecarbon_dir .................................. None + 0: consumed_train_samples .......................... 0 + 0: consumed_train_tokens ........................... 0 + 0: consumed_valid_samples .......................... 0 + 0: contigious_checkpointing ........................ False + 0: cpu_optimizer ................................... False + 0: cpu_torch_adam .................................. False + 0: curriculum_learning ............................. False + 0: data_impl ....................................... mmap + 0: data_parallel_size .............................. 128 + 0: data_path ....................................... None + 0: dataloader_type ................................. single + 0: DDP_impl ........................................ local + 0: decoder_seq_length .............................. None + 0: deepscale ....................................... False + 0: deepscale_config ................................ None + 0: deepspeed ....................................... True + 0: deepspeed_activation_checkpointing .............. False + 0: deepspeed_config ................................ ds_configs/3510898.json + 0: deepspeed_mpi ................................... False + 0: distribute_checkpointed_activations ............. False + 0: distributed_backend ............................. nccl + 0: embed_layernorm ................................. False + 0: embedding_path .................................. None + 0: encoder_seq_length .............................. 2048 + 0: eod_mask_loss ................................... False + 0: eval_interval ................................... 1 + 0: eval_iters ...................................... 100 + 0: eval_only ....................................... True + 0: evidence_data_path .............................. None + 0: exit_duration_in_mins ........................... None + 0: exit_interval ................................... None + 0: ffn_hidden_size ................................. 12288 + 0: finetune ........................................ False + 0: fp16 ............................................ False + 0: fp16_lm_cross_entropy ........................... False + 0: fp32_residual_connection ........................ False + 0: gigaflos_no_embeds .............................. 0 + 0: global_batch_size ............................... 1024 + 0: glu_activation .................................. None + 0: hidden_dropout .................................. 0.1 + 0: hidden_size ..................................... 3072 + 0: hysteresis ...................................... 2 + 0: ict_head_size ................................... None + 0: ict_load ........................................ None + 0: img_dim ......................................... 224 + 0: indexer_batch_size .............................. 128 + 0: indexer_log_interval ............................ 1000 + 0: inference ....................................... False + 0: init_method_std ................................. 0.02 + 0: init_method_xavier_uniform ...................... False + 0: initial_loss_scale .............................. 4294967296 + 0: kill_switch_path ................................ kill-switch-4b284bc4perplexity25val + 0: kv_channels ..................................... 128 + 0: layer_norm_fusion ............................... True + 0: layernorm_epsilon ............................... 1e-05 + 0: lazy_mpu_init ................................... None + 0: load ............................................ checkpoints_4b284bc4perplexity25 + 0: local_rank ...................................... None + 0: log_batch_size_to_tensorboard ................... True + 0: log_interval .................................... 10 + 0: log_learning_rate_to_tensorboard ................ True + 0: log_level ....................................... None + 0: log_level_replica ............................... None + 0: log_loss_scale_to_tensorboard ................... True + 0: log_num_zeros_in_grad ........................... False + 0: log_params_norm ................................. False + 0: log_path ........................................ None + 0: log_timers_to_tensorboard ....................... True + 0: log_validation_ppl_to_tensorboard ............... True + 0: loss_on_targets_only ............................ False + 0: loss_scale ...................................... None + 0: loss_scale_window ............................... 1000 + 0: lr .............................................. 0.0002 + 0: lr_decay_iters .................................. None + 0: lr_decay_samples ................................ 1 + 0: lr_decay_style .................................. cosine + 0: lr_decay_tokens ................................. None + 0: lr_warmup_fraction .............................. None + 0: lr_warmup_iters ................................. 0 + 0: lr_warmup_samples ............................... 0 + 0: make_vocab_size_divisible_by .................... 128 + 0: mask_prob ....................................... 0.15 + 0: masked_softmax_fusion ........................... True + 0: max_position_embeddings ......................... 2048 + 0: mean_noise_span_length .......................... None + 0: memory_centric_tiled_linear ..................... False + 0: merge_file ...................................... gpt2/merges.txt + 0: micro_batch_size ................................ 2 + 0: min_loss_scale .................................. 1.0 + 0: min_lr .......................................... 2e-05 + 0: mmap_warmup ..................................... False + 0: no_load_optim ................................... True + 0: no_load_rng ..................................... None + 0: no_save_optim ................................... None + 0: no_save_rng ..................................... None + 0: noise_density ................................... None + 0: num_attention_heads ............................. 24 + 0: num_channels .................................... 3 + 0: num_classes ..................................... 1000 + 0: num_layers ...................................... 36 + 0: num_layers_per_virtual_pipeline_stage ........... None + 0: num_workers ..................................... 2 + 0: onnx_safe ....................................... None + 0: openai_gelu ..................................... False + 0: optimizer ....................................... adam + 0: optimizer_fusion ................................ True + 0: override_lr_scheduler ........................... True + 0: pad_vocab_size_to ............................... None + 0: params_dtype .................................... torch.bfloat16 + 0: partition_activations ........................... False + 0: patch_dim ....................................... 16 + 0: pipeline_model_parallel_size .................... 1 + 0: position_embedding_type ......................... PositionEmbeddingType.absolute + 0: pp_partition_method ............................. None + 0: profile_backward ................................ False + 0: query_in_block_prob ............................. 0.1 + 0: rampup_batch_size ............................... None + 0: rank ............................................ 0 + 0: remote_device ................................... none + 0: reset_attention_mask ............................ False + 0: reset_position_ids .............................. False + 0: reset_progress .................................. True + 0: retriever_report_topk_accuracies ................ [] + 0: retriever_score_scaling ......................... False + 0: retriever_seq_length ............................ 256 + 0: reweight_loss_based_on_position_frequency ....... False + 0: sample_rate ..................................... 1.0 + 0: save ............................................ checkpoints_4b284bc4perplexity25 + 0: save_interval ................................... 1000 + 0: scatter_gather_tensors_in_pipeline .............. True + 0: scattered_embeddings ............................ False + 0: seed ............................................ 1234 + 0: seq_length ...................................... 2048 + 0: sgd_momentum .................................... 0.9 + 0: short_seq_prob .................................. 0.1 + 0: skip_train_iteration_range ...................... None + 0: split ........................................... None + 0: split_transformers .............................. False + 0: sync_tp_duplicated_parameters ................... False + 0: synchronize_each_layer .......................... False + 0: tensor_model_parallel_size ...................... 2 + 0: tensorboard_dir ................................. tensorboard_4b284bc4perplexity25val + 0: tensorboard_log_interval ........................ 1 + 0: tensorboard_queue_size .......................... 5 + 0: test_weighted_split_paths ....................... None + 0: test_weighted_split_paths_path .................. None + 0: tile_factor ..................................... 1 + 0: titles_data_path ................................ None + 0: tokenizer_name_or_path .......................... None + 0: tokenizer_type .................................. GPT2BPETokenizer + 0: train_iters ..................................... None + 0: train_samples ................................... 1 + 0: train_tokens .................................... None + 0: train_weighted_split_names ...................... ['train'] + 0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document']] + 0: train_weighted_split_paths_path ................. None + 0: train_weighted_split_splits ..................... [['0:1']] + 0: train_weighted_split_weights .................... [['1.0']] + 0: universal_checkpoint ............................ False + 0: use_bnb_optimizer ............................... False + 0: use_checkpoint_lr_scheduler ..................... False + 0: use_contiguous_buffers_in_ddp ................... True + 0: use_cpu_initialization .......................... None + 0: use_one_sent_docs ............................... False + 0: use_pin_memory .................................. False + 0: valid_num_workers ............................... 2 + 0: valid_weighted_split_names ...................... ['validation'] + 0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] + 0: valid_weighted_split_paths_path ................. None + 0: valid_weighted_split_splits ..................... [['0:1']] + 0: valid_weighted_split_weights .................... [['1.0']] + 0: virtual_pipeline_model_parallel_size ............ None + 0: vocab_extra_ids ................................. 0 + 0: vocab_file ...................................... gpt2/vocab.json + 0: weight_decay .................................... 0.1 + 0: world_size ...................................... 256 + 0: zero_allgather_bucket_size ...................... 0.0 + 0: zero_contigious_gradients ....................... False + 0: zero_reduce_bucket_size ......................... 0.0 + 0: zero_reduce_scatter ............................. False + 0: zero_stage ...................................... 0 + 0: -------------------- end of arguments --------------------- + 0: setting number of micro-batches to constant 4 + 0: > building GPT2BPETokenizer tokenizer ... + 0: > padded vocab (size: 50257) with 175 dummy tokens (new size: 50432) + 0: DeepSpeed general environment info: + 0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] + 0: torch version .................... 1.13.0+rocm5.2 + 0: torch cuda version ............... None + 0: torch hip version ................ 5.2.21151-afdc89f8 + 0: nvcc version ..................... None + 0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] + 0: deepspeed info ................... 0.7.5, unknown, unknown + 0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 + 0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** + 0: > initializing torch distributed ... + 0: [2023-05-13 09:37:31,516] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +31: > setting tensorboard ... + 0: > initializing tensor model parallel with size 2 + 0: > initializing pipeline model parallel with size 1 + 0: > setting random seeds to 1234 ... + 0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 + 0: > compiling dataset index builder ... + 0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: make: Nothing to be done for 'default'. + 0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: >>> done with dataset index builder. Compilation time: 0.096 seconds + 0: > compiling and loading fused kernels ... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 102 + 0: [1/1] c++ scaled_masked_softmax_hip.o scaled_masked_softmax_hip.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/opt/rocm/lib -lamdhip64 -o scaled_masked_softmax_cuda.so + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 67 + 0: ninja: no work to do. + 0: >>> done with compiling and loading fused kernels. Compilation time: 32.158 seconds + 0: time to initialize megatron (seconds): 84.328 + 0: [after megatron is initialized] datetime: 2023-05-13 09:38:12 + 0: building GPT model ... + 0: [2023-05-13 09:38:12,467] [INFO] [utils.py:827:see_memory_usage] Before Building Model + 0: [2023-05-13 09:38:12,467] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB + 0: [2023-05-13 09:38:12,468] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.78 GB, percent = 8.1% + 0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None + 0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=0, model=1): 1, ProcessCoord(pipe=0, data=1, model=0): 2, ProcessCoord(pipe=0, data=1, model=1): 3, ProcessCoord(pipe=0, data=2, model=0): 4, ProcessCoord(pipe=0, data=2, model=1): 5, ProcessCoord(pipe=0, data=3, model=0): 6, ProcessCoord(pipe=0, data=3, model=1): 7, ProcessCoord(pipe=0, data=4, model=0): 8, ProcessCoord(pipe=0, data=4, model=1): 9, ProcessCoord(pipe=0, data=5, model=0): 10, ProcessCoord(pipe=0, data=5, model=1): 11, ProcessCoord(pipe=0, data=6, model=0): 12, ProcessCoord(pipe=0, data=6, model=1): 13, ProcessCoord(pipe=0, data=7, model=0): 14, ProcessCoord(pipe=0, data=7, model=1): 15, ProcessCoord(pipe=0, data=8, model=0): 16, ProcessCoord(pipe=0, data=8, model=1): 17, ProcessCoord(pipe=0, data=9, model=0): 18, ProcessCoord(pipe=0, data=9, model=1): 19, ProcessCoord(pipe=0, data=10, model=0): 20, ProcessCoord(pipe=0, data=10, model=1): 21, ProcessCoord(pipe=0, data=11, model=0): 22, ProcessCoord(pipe=0, data + 0: =11, model=1): 23, ProcessCoord(pipe=0, data=12, model=0): 24, ProcessCoord(pipe=0, data=12, model=1): 25, ProcessCoord(pipe=0, data=13, model=0): 26, ProcessCoord(pipe=0, data=13, model=1): 27, ProcessCoord(pipe=0, data=14, model=0): 28, ProcessCoord(pipe=0, data=14, model=1): 29, ProcessCoord(pipe=0, data=15, model=0): 30, ProcessCoord(pipe=0, data=15, model=1): 31, ProcessCoord(pipe=0, data=16, model=0): 32, ProcessCoord(pipe=0, data=16, model=1): 33, ProcessCoord(pipe=0, data=17, model=0): 34, ProcessCoord(pipe=0, data=17, model=1): 35, ProcessCoord(pipe=0, data=18, model=0): 36, ProcessCoord(pipe=0, data=18, model=1): 37, ProcessCoord(pipe=0, data=19, model=0): 38, ProcessCoord(pipe=0, data=19, model=1): 39, ProcessCoord(pipe=0, data=20, model=0): 40, ProcessCoord(pipe=0, data=20, model=1): 41, ProcessCoord(pipe=0, data=21, model=0): 42, ProcessCoord(pipe=0, data=21, model=1): 43, ProcessCoord(pipe=0, data=22, model=0): 44, ProcessCoord(pipe=0, data=22, model=1): 45, ProcessCoord(pipe=0, data=23, model=0 + 0: ): 46, ProcessCoord(pipe=0, data=23, model=1): 47, ProcessCoord(pipe=0, data=24, model=0): 48, ProcessCoord(pipe=0, data=24, model=1): 49, ProcessCoord(pipe=0, data=25, model=0): 50, ProcessCoord(pipe=0, data=25, model=1): 51, ProcessCoord(pipe=0, data=26, model=0): 52, ProcessCoord(pipe=0, data=26, model=1): 53, ProcessCoord(pipe=0, data=27, model=0): 54, ProcessCoord(pipe=0, data=27, model=1): 55, ProcessCoord(pipe=0, data=28, model=0): 56, ProcessCoord(pipe=0, data=28, model=1): 57, ProcessCoord(pipe=0, data=29, model=0): 58, ProcessCoord(pipe=0, data=29, model=1): 59, ProcessCoord(pipe=0, data=30, model=0): 60, ProcessCoord(pipe=0, data=30, model=1): 61, ProcessCoord(pipe=0, data=31, model=0): 62, ProcessCoord(pipe=0, data=31, model=1): 63, ProcessCoord(pipe=0, data=32, model=0): 64, ProcessCoord(pipe=0, data=32, model=1): 65, ProcessCoord(pipe=0, data=33, model=0): 66, ProcessCoord(pipe=0, data=33, model=1): 67, ProcessCoord(pipe=0, data=34, model=0): 68, ProcessCoord(pipe=0, data=34, model=1): 69, Proce + 0: ssCoord(pipe=0, data=35, model=0): 70, ProcessCoord(pipe=0, data=35, model=1): 71, ProcessCoord(pipe=0, data=36, model=0): 72, ProcessCoord(pipe=0, data=36, model=1): 73, ProcessCoord(pipe=0, data=37, model=0): 74, ProcessCoord(pipe=0, data=37, model=1): 75, ProcessCoord(pipe=0, data=38, model=0): 76, ProcessCoord(pipe=0, data=38, model=1): 77, ProcessCoord(pipe=0, data=39, model=0): 78, ProcessCoord(pipe=0, data=39, model=1): 79, ProcessCoord(pipe=0, data=40, model=0): 80, ProcessCoord(pipe=0, data=40, model=1): 81, ProcessCoord(pipe=0, data=41, model=0): 82, ProcessCoord(pipe=0, data=41, model=1): 83, ProcessCoord(pipe=0, data=42, model=0): 84, ProcessCoord(pipe=0, data=42, model=1): 85, ProcessCoord(pipe=0, data=43, model=0): 86, ProcessCoord(pipe=0, data=43, model=1): 87, ProcessCoord(pipe=0, data=44, model=0): 88, ProcessCoord(pipe=0, data=44, model=1): 89, ProcessCoord(pipe=0, data=45, model=0): 90, ProcessCoord(pipe=0, data=45, model=1): 91, ProcessCoord(pipe=0, data=46, model=0): 92, ProcessCoord(pipe + 0: =0, data=46, model=1): 93, ProcessCoord(pipe=0, data=47, model=0): 94, ProcessCoord(pipe=0, data=47, model=1): 95, ProcessCoord(pipe=0, data=48, model=0): 96, ProcessCoord(pipe=0, data=48, model=1): 97, ProcessCoord(pipe=0, data=49, model=0): 98, ProcessCoord(pipe=0, data=49, model=1): 99, ProcessCoord(pipe=0, data=50, model=0): 100, ProcessCoord(pipe=0, data=50, model=1): 101, ProcessCoord(pipe=0, data=51, model=0): 102, ProcessCoord(pipe=0, data=51, model=1): 103, ProcessCoord(pipe=0, data=52, model=0): 104, ProcessCoord(pipe=0, data=52, model=1): 105, ProcessCoord(pipe=0, data=53, model=0): 106, ProcessCoord(pipe=0, data=53, model=1): 107, ProcessCoord(pipe=0, data=54, model=0): 108, ProcessCoord(pipe=0, data=54, model=1): 109, ProcessCoord(pipe=0, data=55, model=0): 110, ProcessCoord(pipe=0, data=55, model=1): 111, ProcessCoord(pipe=0, data=56, model=0): 112, ProcessCoord(pipe=0, data=56, model=1): 113, ProcessCoord(pipe=0, data=57, model=0): 114, ProcessCoord(pipe=0, data=57, model=1): 115, ProcessCoord( + 0: pipe=0, data=58, model=0): 116, ProcessCoord(pipe=0, data=58, model=1): 117, ProcessCoord(pipe=0, data=59, model=0): 118, ProcessCoord(pipe=0, data=59, model=1): 119, ProcessCoord(pipe=0, data=60, model=0): 120, ProcessCoord(pipe=0, data=60, model=1): 121, ProcessCoord(pipe=0, data=61, model=0): 122, ProcessCoord(pipe=0, data=61, model=1): 123, ProcessCoord(pipe=0, data=62, model=0): 124, ProcessCoord(pipe=0, data=62, model=1): 125, ProcessCoord(pipe=0, data=63, model=0): 126, ProcessCoord(pipe=0, data=63, model=1): 127, ProcessCoord(pipe=0, data=64, model=0): 128, ProcessCoord(pipe=0, data=64, model=1): 129, ProcessCoord(pipe=0, data=65, model=0): 130, ProcessCoord(pipe=0, data=65, model=1): 131, ProcessCoord(pipe=0, data=66, model=0): 132, ProcessCoord(pipe=0, data=66, model=1): 133, ProcessCoord(pipe=0, data=67, model=0): 134, ProcessCoord(pipe=0, data=67, model=1): 135, ProcessCoord(pipe=0, data=68, model=0): 136, ProcessCoord(pipe=0, data=68, model=1): 137, ProcessCoord(pipe=0, data=69, model=0): 138, Pr + 0: ocessCoord(pipe=0, data=69, model=1): 139, ProcessCoord(pipe=0, data=70, model=0): 140, ProcessCoord(pipe=0, data=70, model=1): 141, ProcessCoord(pipe=0, data=71, model=0): 142, ProcessCoord(pipe=0, data=71, model=1): 143, ProcessCoord(pipe=0, data=72, model=0): 144, ProcessCoord(pipe=0, data=72, model=1): 145, ProcessCoord(pipe=0, data=73, model=0): 146, ProcessCoord(pipe=0, data=73, model=1): 147, ProcessCoord(pipe=0, data=74, model=0): 148, ProcessCoord(pipe=0, data=74, model=1): 149, ProcessCoord(pipe=0, data=75, model=0): 150, ProcessCoord(pipe=0, data=75, model=1): 151, ProcessCoord(pipe=0, data=76, model=0): 152, ProcessCoord(pipe=0, data=76, model=1): 153, ProcessCoord(pipe=0, data=77, model=0): 154, ProcessCoord(pipe=0, data=77, model=1): 155, ProcessCoord(pipe=0, data=78, model=0): 156, ProcessCoord(pipe=0, data=78, model=1): 157, ProcessCoord(pipe=0, data=79, model=0): 158, ProcessCoord(pipe=0, data=79, model=1): 159, ProcessCoord(pipe=0, data=80, model=0): 160, ProcessCoord(pipe=0, data=80, model= + 0: 1): 161, ProcessCoord(pipe=0, data=81, model=0): 162, ProcessCoord(pipe=0, data=81, model=1): 163, ProcessCoord(pipe=0, data=82, model=0): 164, ProcessCoord(pipe=0, data=82, model=1): 165, ProcessCoord(pipe=0, data=83, model=0): 166, ProcessCoord(pipe=0, data=83, model=1): 167, ProcessCoord(pipe=0, data=84, model=0): 168, ProcessCoord(pipe=0, data=84, model=1): 169, ProcessCoord(pipe=0, data=85, model=0): 170, ProcessCoord(pipe=0, data=85, model=1): 171, ProcessCoord(pipe=0, data=86, model=0): 172, ProcessCoord(pipe=0, data=86, model=1): 173, ProcessCoord(pipe=0, data=87, model=0): 174, ProcessCoord(pipe=0, data=87, model=1): 175, ProcessCoord(pipe=0, data=88, model=0): 176, ProcessCoord(pipe=0, data=88, model=1): 177, ProcessCoord(pipe=0, data=89, model=0): 178, ProcessCoord(pipe=0, data=89, model=1): 179, ProcessCoord(pipe=0, data=90, model=0): 180, ProcessCoord(pipe=0, data=90, model=1): 181, ProcessCoord(pipe=0, data=91, model=0): 182, ProcessCoord(pipe=0, data=91, model=1): 183, ProcessCoord(pipe=0, data + 0: =92, model=0): 184, ProcessCoord(pipe=0, data=92, model=1): 185, ProcessCoord(pipe=0, data=93, model=0): 186, ProcessCoord(pipe=0, data=93, model=1): 187, ProcessCoord(pipe=0, data=94, model=0): 188, ProcessCoord(pipe=0, data=94, model=1): 189, ProcessCoord(pipe=0, data=95, model=0): 190, ProcessCoord(pipe=0, data=95, model=1): 191, ProcessCoord(pipe=0, data=96, model=0): 192, ProcessCoord(pipe=0, data=96, model=1): 193, ProcessCoord(pipe=0, data=97, model=0): 194, ProcessCoord(pipe=0, data=97, model=1): 195, ProcessCoord(pipe=0, data=98, model=0): 196, ProcessCoord(pipe=0, data=98, model=1): 197, ProcessCoord(pipe=0, data=99, model=0): 198, ProcessCoord(pipe=0, data=99, model=1): 199, ProcessCoord(pipe=0, data=100, model=0): 200, ProcessCoord(pipe=0, data=100, model=1): 201, ProcessCoord(pipe=0, data=101, model=0): 202, ProcessCoord(pipe=0, data=101, model=1): 203, ProcessCoord(pipe=0, data=102, model=0): 204, ProcessCoord(pipe=0, data=102, model=1): 205, ProcessCoord(pipe=0, data=103, model=0): 206, Process + 0: Coord(pipe=0, data=103, model=1): 207, ProcessCoord(pipe=0, data=104, model=0): 208, ProcessCoord(pipe=0, data=104, model=1): 209, ProcessCoord(pipe=0, data=105, model=0): 210, ProcessCoord(pipe=0, data=105, model=1): 211, ProcessCoord(pipe=0, data=106, model=0): 212, ProcessCoord(pipe=0, data=106, model=1): 213, ProcessCoord(pipe=0, data=107, model=0): 214, ProcessCoord(pipe=0, data=107, model=1): 215, ProcessCoord(pipe=0, data=108, model=0): 216, ProcessCoord(pipe=0, data=108, model=1): 217, ProcessCoord(pipe=0, data=109, model=0): 218, ProcessCoord(pipe=0, data=109, model=1): 219, ProcessCoord(pipe=0, data=110, model=0): 220, ProcessCoord(pipe=0, data=110, model=1): 221, ProcessCoord(pipe=0, data=111, model=0): 222, ProcessCoord(pipe=0, data=111, model=1): 223, ProcessCoord(pipe=0, data=112, model=0): 224, ProcessCoord(pipe=0, data=112, model=1): 225, ProcessCoord(pipe=0, data=113, model=0): 226, ProcessCoord(pipe=0, data=113, model=1): 227, ProcessCoord(pipe=0, data=114, model=0): 228, ProcessCoord(pipe=0 + 0: , data=114, model=1): 229, ProcessCoord(pipe=0, data=115, model=0): 230, ProcessCoord(pipe=0, data=115, model=1): 231, ProcessCoord(pipe=0, data=116, model=0): 232, ProcessCoord(pipe=0, data=116, model=1): 233, ProcessCoord(pipe=0, data=117, model=0): 234, ProcessCoord(pipe=0, data=117, model=1): 235, ProcessCoord(pipe=0, data=118, model=0): 236, ProcessCoord(pipe=0, data=118, model=1): 237, ProcessCoord(pipe=0, data=119, model=0): 238, ProcessCoord(pipe=0, data=119, model=1): 239, ProcessCoord(pipe=0, data=120, model=0): 240, ProcessCoord(pipe=0, data=120, model=1): 241, ProcessCoord(pipe=0, data=121, model=0): 242, ProcessCoord(pipe=0, data=121, model=1): 243, ProcessCoord(pipe=0, data=122, model=0): 244, ProcessCoord(pipe=0, data=122, model=1): 245, ProcessCoord(pipe=0, data=123, model=0): 246, ProcessCoord(pipe=0, data=123, model=1): 247, ProcessCoord(pipe=0, data=124, model=0): 248, ProcessCoord(pipe=0, data=124, model=1): 249, ProcessCoord(pipe=0, data=125, model=0): 250, ProcessCoord(pipe=0, data=125, + 0: model=1): 251, ProcessCoord(pipe=0, data=126, model=0): 252, ProcessCoord(pipe=0, data=126, model=1): 253, ProcessCoord(pipe=0, data=127, model=0): 254, ProcessCoord(pipe=0, data=127, model=1): 255} + 0: [2023-05-13 09:38:17,921] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer + 0: stage=0 layers=43 + 0: 0: _to_float16 + 0: 1: EmbeddingPipe + 0: 2: + 0: 3: ParallelTransformerLayerPipe + 0: 4: ParallelTransformerLayerPipe + 0: 5: ParallelTransformerLayerPipe + 0: 6: ParallelTransformerLayerPipe + 0: 7: ParallelTransformerLayerPipe + 0: 8: ParallelTransformerLayerPipe + 0: 9: ParallelTransformerLayerPipe + 0: 10: ParallelTransformerLayerPipe + 0: 11: ParallelTransformerLayerPipe + 0: 12: ParallelTransformerLayerPipe + 0: 13: ParallelTransformerLayerPipe + 0: 14: ParallelTransformerLayerPipe + 0: 15: ParallelTransformerLayerPipe + 0: 16: ParallelTransformerLayerPipe + 0: 17: ParallelTransformerLayerPipe + 0: 18: ParallelTransformerLayerPipe + 0: 19: ParallelTransformerLayerPipe + 0: 20: ParallelTransformerLayerPipe + 0: 21: ParallelTransformerLayerPipe + 0: 22: ParallelTransformerLayerPipe + 0: 23: ParallelTransformerLayerPipe + 0: 24: ParallelTransformerLayerPipe + 0: 25: ParallelTransformerLayerPipe + 0: 26: ParallelTransformerLayerPipe + 0: 27: ParallelTransformerLayerPipe + 0: 28: ParallelTransformerLayerPipe + 0: 29: ParallelTransformerLayerPipe + 0: 30: ParallelTransformerLayerPipe + 0: 31: ParallelTransformerLayerPipe + 0: 32: ParallelTransformerLayerPipe + 0: 33: ParallelTransformerLayerPipe + 0: 34: ParallelTransformerLayerPipe + 0: 35: ParallelTransformerLayerPipe + 0: 36: ParallelTransformerLayerPipe + 0: 37: ParallelTransformerLayerPipe + 0: 38: ParallelTransformerLayerPipe + 0: 39: undo + 0: 40: MixedFusedLayerNorm + 0: 41: EmbeddingPipe + 0: 42: float16_to_fp32 + 0: loss: CrossEntropy + 0: [2023-05-13 09:38:18,397] [INFO] [utils.py:827:see_memory_usage] After Building Model + 0: [2023-05-13 09:38:18,397] [INFO] [utils.py:828:see_memory_usage] MA 4.0 GB Max_MA 4.0 GB CA 4.04 GB Max_CA 4 GB + 0: [2023-05-13 09:38:18,398] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.0 GB, percent = 8.1% + 0: setting training iterations to 0 + 0: > learning rate decay style: cosine + 0: DeepSpeed is enabled. + 0: [2023-05-13 09:38:18,401] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown + 0: [2023-05-13 09:38:23,901] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False + 0: [2023-05-13 09:38:23,901] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer + 0: [2023-05-13 09:38:23,901] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer + 0: [2023-05-13 09:38:23,922] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam + 0: [2023-05-13 09:38:23,922] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer + 0: [2023-05-13 09:38:24,044] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer + 0: [2023-05-13 09:38:24,044] [INFO] [utils.py:828:see_memory_usage] MA 3.99 GB Max_MA 4.01 GB CA 4.05 GB Max_CA 4 GB + 0: [2023-05-13 09:38:24,044] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.77 GB, percent = 8.3% +14: ninja: no work to do. +14: Time to load utils op: 0.17441368103027344 seconds + 0: ninja: no work to do. + 0: Time to load utils op: 0.18263936042785645 seconds +14: Time to load utils op: 0.0006191730499267578 seconds + 0: Time to load utils op: 0.000659942626953125 seconds +27: Time to load utils op: 0.30866217613220215 seconds + 0: Time to load utils op: 0.606006383895874 seconds + 0: Time to load utils op: 0.6052193641662598 seconds + 0: Time to load utils op: 0.6060769557952881 seconds + 0: Time to load utils op: 0.40255212783813477 seconds + 0: Time to load utils op: 0.4021604061126709 seconds + 0: Time to load utils op: 0.5030276775360107 seconds + 0: Time to load utils op: 0.40219926834106445 seconds + 2: Time to load utils op: 0.6115593910217285 seconds + 2: Time to load utils op: 0.6113007068634033 secondsTime to load utils op: 0.6112751960754395 seconds + 2: + 2: Time to load utils op: 0.4071831703186035 seconds + 2: Time to load utils op: 0.4079442024230957 secondsTime to load utils op: 0.4075651168823242 seconds + 2: Time to load utils op: 0.6121490001678467 seconds + 2: Time to load utils op: 0.40743374824523926 seconds + 2: + 4: Time to load utils op: 0.6125469207763672 secondsTime to load utils op: 0.40870189666748047 secondsTime to load utils op: 0.40841031074523926 seconds + 4: + 4: + 4: Time to load utils op: 0.6124475002288818 seconds + 4: Time to load utils op: 0.612591028213501 secondsTime to load utils op: 0.612445592880249 seconds + 4: + 4: Time to load utils op: 0.40755677223205566 seconds + 4: Time to load utils op: 0.4078943729400635 seconds + 5: Time to load utils op: 0.4074671268463135 secondsTime to load utils op: 0.4085395336151123 seconds + 5: + 5: Time to load utils op: 0.4068882465362549 seconds + 5: Time to load utils op: 0.6131782531738281 secondsTime to load utils op: 0.4086289405822754 seconds + 5: + 5: Time to load utils op: 0.6123228073120117 secondsTime to load utils op: 0.6129868030548096 seconds + 5: Time to load utils op: 0.6130867004394531 seconds + 5: + 1: Time to load utils op: 0.4108541011810303 seconds + 1: Time to load utils op: 0.611191987991333 secondsTime to load utils op: 0.6112020015716553 seconds + 1: + 1: Time to load utils op: 0.41100597381591797 seconds + 1: Time to load utils op: 0.41101717948913574 secondsTime to load utils op: 0.6112267971038818 seconds + 1: + 1: Time to load utils op: 0.6112298965454102 seconds + 1: Time to load utils op: 0.4107842445373535 seconds + 7: Time to load utils op: 0.6120047569274902 seconds + 7: Time to load utils op: 0.6123538017272949 seconds + 7: Time to load utils op: 0.4076826572418213 seconds + 7: Time to load utils op: 0.40865111351013184 seconds + 7: Time to load utils op: 0.40750837326049805 seconds + 7: Time to load utils op: 0.408522367477417 secondsTime to load utils op: 0.6133449077606201 seconds + 7: + 8: Time to load utils op: 0.6125645637512207 seconds + 8: Time to load utils op: 0.6119341850280762 seconds + 8: Time to load utils op: 0.6114668846130371 seconds + 7: Time to load utils op: 0.6125044822692871 seconds + 8: Time to load utils op: 0.4080684185028076 seconds + 8: Time to load utils op: 0.6132638454437256 seconds + 8: Time to load utils op: 0.40763187408447266 seconds + 8: Time to load utils op: 0.4078178405761719 secondsTime to load utils op: 0.40834808349609375 seconds + 8: + 3: Time to load utils op: 0.6114821434020996 seconds + 3: Time to load utils op: 0.6114828586578369 seconds + 3: Time to load utils op: 0.611534833908081 seconds + 3: Time to load utils op: 0.4101402759552002 seconds + 3: Time to load utils op: 0.41020894050598145 seconds + 3: Time to load utils op: 0.4103705883026123 seconds + 3: Time to load utils op: 0.41048574447631836 secondsTime to load utils op: 0.6115603446960449 seconds + 3: + 6: Time to load utils op: 0.41061973571777344 seconds + 6: Time to load utils op: 0.6113643646240234 seconds + 6: Time to load utils op: 0.40996408462524414 seconds + 6: Time to load utils op: 0.4100050926208496 secondsTime to load utils op: 0.41017746925354004 seconds + 6: + 6: Time to load utils op: 0.6114358901977539 secondsTime to load utils op: 0.6114373207092285 seconds + 6: Time to load utils op: 0.6114459037780762 seconds + 6: +14: Time to load utils op: 0.4085087776184082 seconds +15: Time to load utils op: 0.40771055221557617 seconds +15: Time to load utils op: 0.6120033264160156 seconds +15: Time to load utils op: 0.6133396625518799 seconds + 9: Time to load utils op: 0.41080212593078613 secondsTime to load utils op: 0.6112537384033203 seconds + 9: + 9: Time to load utils op: 0.4106922149658203 seconds + 9: Time to load utils op: 0.4107534885406494 seconds + 9: Time to load utils op: 0.6113221645355225 seconds + 9: Time to load utils op: 0.6113219261169434 secondsTime to load utils op: 0.4107081890106201 seconds + 9: +15: Time to load utils op: 0.6117823123931885 secondsTime to load utils op: 0.6115767955780029 seconds +15: + 9: Time to load utils op: 0.6113271713256836 seconds +15: Time to load utils op: 0.40760207176208496 secondsTime to load utils op: 0.4079883098602295 seconds +15: +15: Time to load utils op: 0.4082145690917969 seconds +10: Time to load utils op: 0.6114299297332764 secondsTime to load utils op: 0.41072678565979004 secondsTime to load utils op: 0.6114282608032227 seconds +10: +10: +10: Time to load utils op: 0.6114423274993896 seconds +10: Time to load utils op: 0.4106175899505615 seconds +10: Time to load utils op: 0.41066503524780273 secondsTime to load utils op: 0.6114704608917236 seconds +10: +10: Time to load utils op: 0.4105567932128906 seconds +11: Time to load utils op: 0.41109418869018555 seconds +11: Time to load utils op: 0.6125752925872803 seconds +11: Time to load utils op: 0.6125948429107666 seconds +11: Time to load utils op: 0.6125993728637695 secondsTime to load utils op: 0.41112637519836426 seconds +11: +11: Time to load utils op: 0.41100120544433594 seconds +11: Time to load utils op: 0.41092395782470703 seconds +11: Time to load utils op: 0.6126265525817871 seconds +12: Time to load utils op: 0.6115579605102539 secondsTime to load utils op: 0.4107062816619873 seconds +12: +12: Time to load utils op: 0.6115784645080566 seconds +12: Time to load utils op: 0.410322904586792 seconds +12: Time to load utils op: 0.6116137504577637 seconds +12: Time to load utils op: 0.410707950592041 secondsTime to load utils op: 0.4107673168182373 seconds +12: +12: Time to load utils op: 0.6116366386413574 seconds +13: Time to load utils op: 0.6149241924285889 secondsTime to load utils op: 0.41184353828430176 seconds +13: +13: Time to load utils op: 0.41178441047668457 seconds +13: Time to load utils op: 0.41216111183166504 seconds +13: Time to load utils op: 0.6152806282043457 seconds +13: Time to load utils op: 0.6149027347564697 secondsTime to load utils op: 0.411146879196167 seconds +13: +13: Time to load utils op: 0.6150574684143066 seconds +17: Time to load utils op: 0.408963680267334 seconds +17: Time to load utils op: 0.6091151237487793 seconds +17: Time to load utils op: 0.40895843505859375 seconds +17: Time to load utils op: 0.40901827812194824 secondsTime to load utils op: 0.6091570854187012 seconds +17: +17: Time to load utils op: 0.6091709136962891 secondsTime to load utils op: 0.40883731842041016 seconds +17: +17: Time to load utils op: 0.6091830730438232 seconds +16: Time to load utils op: 0.41100502014160156 seconds +16: Time to load utils op: 0.6103923320770264 seconds +16: Time to load utils op: 0.4111602306365967 seconds +16: Time to load utils op: 0.6104707717895508 seconds +16: Time to load utils op: 0.41123199462890625 secondsTime to load utils op: 0.4109959602355957 seconds +16: +16: Time to load utils op: 0.6104745864868164 seconds +16: Time to load utils op: 0.6104855537414551 seconds +27: Time to load utils op: 0.4048340320587158 seconds +27: Time to load utils op: 0.6052675247192383 seconds +27: Time to load utils op: 0.4049406051635742 seconds +18: Time to load utils op: 0.4102511405944824 secondsTime to load utils op: 0.6111767292022705 seconds +18: +18: Time to load utils op: 0.6112630367279053 secondsTime to load utils op: 0.6112563610076904 seconds +18: +18: Time to load utils op: 0.41013264656066895 seconds +18: Time to load utils op: 0.6112837791442871 seconds +18: Time to load utils op: 0.4103975296020508 secondsTime to load utils op: 0.41028571128845215 seconds +18: +27: Time to load utils op: 0.6060450077056885 seconds +27: Time to load utils op: 0.6062929630279541 seconds +27: Time to load utils op: 0.405773401260376 seconds +20: Time to load utils op: 0.6098153591156006 secondsTime to load utils op: 0.4096851348876953 secondsTime to load utils op: 0.6098177433013916 seconds +20: +20: +20: Time to load utils op: 0.6098499298095703 seconds +20: Time to load utils op: 0.6098518371582031 seconds +20: Time to load utils op: 0.4101259708404541 seconds +20: Time to load utils op: 0.4093179702758789 secondsTime to load utils op: 0.40978026390075684 seconds +20: +27: Time to load utils op: 0.6067254543304443 seconds +14: Time to load utils op: 0.6178364753723145 seconds +21: Time to load utils op: 0.4090402126312256 seconds +21: Time to load utils op: 0.6100122928619385 seconds +21: Time to load utils op: 0.4098362922668457 seconds +21: Time to load utils op: 0.40972018241882324 seconds +21: Time to load utils op: 0.6101000308990479 seconds +21: Time to load utils op: 0.40936732292175293 secondsTime to load utils op: 0.6101081371307373 seconds +21: Time to load utils op: 0.6101047992706299 seconds +21: +19: Time to load utils op: 0.41194629669189453 seconds +19: Time to load utils op: 0.41199445724487305 seconds + 0: Time to load utils op: 0.00055694580078125 seconds +19: Time to load utils op: 0.41217589378356934 seconds +19: Time to load utils op: 0.6120016574859619 seconds +19: Time to load utils op: 0.6120178699493408 seconds +19: Time to load utils op: 0.4121668338775635 secondsTime to load utils op: 0.6120090484619141 secondsTime to load utils op: 0.6120038032531738 seconds +19: +19: + 0: Time to load utils op: 0.00041294097900390625 seconds + 0: Time to load utils op: 0.00040149688720703125 seconds + 0: Time to load utils op: 0.0004127025604248047 seconds + 0: Time to load utils op: 0.00038886070251464844 seconds +22: Time to load utils op: 0.611717939376831 seconds +22: Time to load utils op: 0.41126132011413574 seconds +22: Time to load utils op: 0.41131019592285156 seconds +22: Time to load utils op: 0.4112589359283447 seconds +22: Time to load utils op: 0.6117877960205078 seconds +22: Time to load utils op: 0.611802339553833 seconds +22: Time to load utils op: 0.41115665435791016 seconds +22: Time to load utils op: 0.6118125915527344 seconds +26: Time to load utils op: 0.6102700233459473 seconds +26: Time to load utils op: 0.6102814674377441 seconds +26: Time to load utils op: 0.4097926616668701 seconds +26: Time to load utils op: 0.610314130783081 seconds +26: Time to load utils op: 0.4097893238067627 secondsTime to load utils op: 0.40996837615966797 secondsTime to load utils op: 0.6103241443634033 seconds +26: +26: +26: Time to load utils op: 0.4097921848297119 seconds +23: Time to load utils op: 0.4111654758453369 secondsTime to load utils op: 0.6120564937591553 secondsTime to load utils op: 0.6120486259460449 seconds +23: +23: + 0: Time to load utils op: 0.0004451274871826172 seconds +23: Time to load utils op: 0.411205530166626 secondsTime to load utils op: 0.41109204292297363 seconds +23: +23: Time to load utils op: 0.6120965480804443 seconds +23: Time to load utils op: 0.4112374782562256 secondsTime to load utils op: 0.6121039390563965 seconds +23: +24: Time to load utils op: 0.6125853061676025 seconds +24: Time to load utils op: 0.41136765480041504 secondsTime to load utils op: 0.6125831604003906 seconds +24: +24: Time to load utils op: 0.6126058101654053 seconds +24: Time to load utils op: 0.6126134395599365 secondsTime to load utils op: 0.41008830070495605 seconds +24: Time to load utils op: 0.4112863540649414 seconds +24: +24: Time to load utils op: 0.41110801696777344 seconds +25: Time to load utils op: 0.4108867645263672 secondsTime to load utils op: 0.6109223365783691 seconds +25: +25: Time to load utils op: 0.4110386371612549 seconds +25: Time to load utils op: 0.6109607219696045 seconds +25: Time to load utils op: 0.6109790802001953 secondsTime to load utils op: 0.6109788417816162 seconds +25: +25: Time to load utils op: 0.4111900329589844 seconds +25: Time to load utils op: 0.41118550300598145 seconds +14: Time to load utils op: 0.6232473850250244 seconds +29: Time to load utils op: 0.4101097583770752 secondsTime to load utils op: 0.6100819110870361 seconds +29: +29: Time to load utils op: 0.6100888252258301 seconds +28: Time to load utils op: 0.4104459285736084 seconds +28: Time to load utils op: 0.6111578941345215 seconds +29: Time to load utils op: 0.4100208282470703 seconds +29: Time to load utils op: 0.6101343631744385 seconds +29: Time to load utils op: 0.6101384162902832 seconds +28: Time to load utils op: 0.4106616973876953 seconds +29: Time to load utils op: 0.4102027416229248 seconds +29: Time to load utils op: 0.4102025032043457 seconds +28: Time to load utils op: 0.41051292419433594 seconds +28: Time to load utils op: 0.6112227439880371 seconds +28: Time to load utils op: 0.6112289428710938 secondsTime to load utils op: 0.611229658126831 seconds +28: +28: Time to load utils op: 0.41062331199645996 seconds +31: Time to load utils op: 0.6134016513824463 seconds +31: Time to load utils op: 0.4104137420654297 seconds +31: Time to load utils op: 0.4108700752258301 seconds +31: Time to load utils op: 0.6134357452392578 seconds +31: Time to load utils op: 0.4103505611419678 seconds +31: Time to load utils op: 0.6134788990020752 secondsTime to load utils op: 0.41037726402282715 seconds +31: +31: Time to load utils op: 0.6136751174926758 seconds +30: Time to load utils op: 0.41065120697021484 secondsTime to load utils op: 0.41065001487731934 secondsTime to load utils op: 0.6091439723968506 seconds +30: +30: +30: Time to load utils op: 0.6091527938842773 seconds +30: Time to load utils op: 0.609168291091919 seconds +30: Time to load utils op: 0.4106478691101074 secondsTime to load utils op: 0.609180212020874 seconds +30: +30: Time to load utils op: 0.4105682373046875 seconds +14: Time to load utils op: 0.42352724075317383 seconds +14: Time to load utils op: 0.42485928535461426 seconds +14: Time to load utils op: 0.42575645446777344 seconds +14: Time to load utils op: 0.630467414855957 seconds +14: Time to load utils op: 0.002577066421508789 seconds +14: Time to load utils op: 0.0026412010192871094 seconds +14: Time to load utils op: 0.0003750324249267578 seconds +14: Time to load utils op: 0.00039505958557128906 seconds +14: Time to load utils op: 0.00039124488830566406 seconds +14: Time to load utils op: 0.0003421306610107422 seconds +14: Time to load utils op: 0.0003867149353027344 seconds + 0: [2023-05-13 09:38:24,671] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 + 0: [2023-05-13 09:38:24,672] [INFO] [utils.py:828:see_memory_usage] MA 3.99 GB Max_MA 3.99 GB CA 4.05 GB Max_CA 4 GB + 0: [2023-05-13 09:38:24,672] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.9 GB, percent = 8.3% +27: Time to load utils op: 0.0008127689361572266 seconds +27: Time to load utils op: 0.0008158683776855469 secondsTime to load utils op: 0.0007851123809814453 seconds +27: +27: Time to load utils op: 0.0008070468902587891 seconds +27: Time to load utils op: 0.0008802413940429688 seconds +27: Time to load utils op: 0.0008549690246582031 seconds +27: Time to load utils op: 0.0008072853088378906 seconds +27: Time to load utils op: 0.0009109973907470703 seconds + 0: [2023-05-13 09:38:24,786] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 + 0: [2023-05-13 09:38:24,787] [INFO] [utils.py:828:see_memory_usage] MA 8.04 GB Max_MA 8.04 GB CA 10.11 GB Max_CA 10 GB + 0: [2023-05-13 09:38:24,787] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.9 GB, percent = 8.3% + 8: Time to load utils op: 0.0005612373352050781 secondsTime to load utils op: 0.0005686283111572266 seconds + 8: + 8: Time to load utils op: 0.0005631446838378906 secondsTime to load utils op: 0.0005955696105957031 seconds + 8: + 8: Time to load utils op: 0.0006628036499023438 secondsTime to load utils op: 0.0006237030029296875 seconds + 8: + 8: Time to load utils op: 0.0006256103515625 seconds + 8: Time to load utils op: 0.0006515979766845703 seconds + 7: Time to load utils op: 0.0005080699920654297 seconds + 7: Time to load utils op: 0.0004124641418457031 seconds + 7: Time to load utils op: 0.0005295276641845703 seconds + 7: Time to load utils op: 0.0004208087921142578 secondsTime to load utils op: 0.000560760498046875 seconds + 7: + 7: Time to load utils op: 0.0005724430084228516 seconds + 7: Time to load utils op: 0.0005464553833007812 seconds + 7: Time to load utils op: 0.0006015300750732422 seconds + 4: Time to load utils op: 0.0005552768707275391 seconds + 4: Time to load utils op: 0.0005731582641601562 seconds + 4: Time to load utils op: 0.0009920597076416016 seconds + 4: Time to load utils op: 0.0010476112365722656 seconds + 4: Time to load utils op: 0.0011096000671386719 seconds + 4: Time to load utils op: 0.0010845661163330078 seconds + 4: Time to load utils op: 0.0010733604431152344 seconds + 4: Time to load utils op: 0.0011453628540039062 seconds + 5: Time to load utils op: 0.0005686283111572266 secondsTime to load utils op: 0.0005726814270019531 seconds + 5: + 5: Time to load utils op: 0.0004401206970214844 seconds + 5: Time to load utils op: 0.0009176731109619141 seconds + 5: Time to load utils op: 0.0008447170257568359 secondsTime to load utils op: 0.0008263587951660156 seconds + 5: + 5: Time to load utils op: 0.0008671283721923828 seconds + 5: Time to load utils op: 0.0008988380432128906 seconds +15: Time to load utils op: 0.0005099773406982422 seconds +15: Time to load utils op: 0.0004444122314453125 seconds +31: Time to load utils op: 0.0012769699096679688 seconds +15: Time to load utils op: 0.0004544258117675781 seconds +15: Time to load utils op: 0.0005831718444824219 seconds +15: Time to load utils op: 0.0006036758422851562 seconds +15: Time to load utils op: 0.0005931854248046875 seconds +15: Time to load utils op: 0.0004773139953613281 seconds +15: Time to load utils op: 0.0006346702575683594 seconds +31: Time to load utils op: 0.0016002655029296875 seconds +31: Time to load utils op: 0.0016872882843017578 seconds +31: Time to load utils op: 0.0017626285552978516 seconds +31: Time to load utils op: 0.001779317855834961 seconds +31: Time to load utils op: 0.0018315315246582031 seconds +31: Time to load utils op: 0.0017943382263183594 seconds +31: Time to load utils op: 0.001819610595703125 seconds +16: Time to load utils op: 0.0004992485046386719 seconds +16: Time to load utils op: 0.0004057884216308594 secondsTime to load utils op: 0.0004336833953857422 secondsTime to load utils op: 0.00041174888610839844 seconds +16: +16: +16: Time to load utils op: 0.0004181861877441406 seconds +16: Time to load utils op: 0.00042748451232910156 seconds +16: Time to load utils op: 0.0005447864532470703 seconds +21: Time to load utils op: 0.0005466938018798828 seconds +21: Time to load utils op: 0.0005064010620117188 seconds +21: Time to load utils op: 0.0005347728729248047 seconds +11: Time to load utils op: 0.0007796287536621094 seconds +16: Time to load utils op: 0.0008912086486816406 seconds + 3: Time to load utils op: 0.0011615753173828125 seconds +21: Time to load utils op: 0.000560760498046875 seconds +21: Time to load utils op: 0.0006134510040283203 secondsTime to load utils op: 0.0006072521209716797 seconds +21: +21: Time to load utils op: 0.0005962848663330078 seconds + 3: Time to load utils op: 0.0012853145599365234 seconds +11: Time to load utils op: 0.0009317398071289062 seconds +21: Time to load utils op: 0.0007059574127197266 seconds +11: Time to load utils op: 0.0011844635009765625 seconds + 3: Time to load utils op: 0.0014753341674804688 secondsTime to load utils op: 0.0014996528625488281 seconds + 3: + 3: Time to load utils op: 0.00151824951171875 seconds + 3: Time to load utils op: 0.0015475749969482422 seconds + 3: Time to load utils op: 0.0015251636505126953 seconds + 3: Time to load utils op: 0.001575469970703125 seconds +11: Time to load utils op: 0.0012814998626708984 seconds +11: Time to load utils op: 0.0013339519500732422 seconds +11: Time to load utils op: 0.0012750625610351562 seconds +11: Time to load utils op: 0.0012729167938232422 seconds +11: Time to load utils op: 0.001348257064819336 seconds +28: Time to load utils op: 0.0006999969482421875 seconds +28: Time to load utils op: 0.0007112026214599609 seconds +28: Time to load utils op: 0.0008602142333984375 seconds +28: Time to load utils op: 0.0009586811065673828 seconds +28: Time to load utils op: 0.0009465217590332031 seconds +28: Time to load utils op: 0.0009279251098632812 seconds +28: Time to load utils op: 0.0009255409240722656 seconds +28: Time to load utils op: 0.0010023117065429688 seconds + 6: Time to load utils op: 0.0005536079406738281 seconds + 6: Time to load utils op: 0.0008089542388916016 seconds + 6: Time to load utils op: 0.0007300376892089844 seconds + 6: Time to load utils op: 0.0007817745208740234 secondsTime to load utils op: 0.0008521080017089844 seconds + 6: + 6: Time to load utils op: 0.0009307861328125 secondsTime to load utils op: 0.0009591579437255859 seconds + 6: + 6: Time to load utils op: 0.0007760524749755859 seconds +23: Time to load utils op: 0.0007038116455078125 seconds +23: Time to load utils op: 0.0007712841033935547 seconds +23: Time to load utils op: 0.0007379055023193359 seconds +19: Time to load utils op: 0.0007183551788330078 secondsTime to load utils op: 0.0007266998291015625 seconds +19: +19: Time to load utils op: 0.0007846355438232422 seconds +19: Time to load utils op: 0.0007090568542480469 seconds +23: Time to load utils op: 0.001138925552368164 seconds +23: Time to load utils op: 0.0011148452758789062 seconds +19: Time to load utils op: 0.0008502006530761719 seconds +23: Time to load utils op: 0.0009975433349609375 seconds +19: Time to load utils op: 0.0008196830749511719 seconds +19: Time to load utils op: 0.0008327960968017578 seconds +23: Time to load utils op: 0.0010395050048828125 secondsTime to load utils op: 0.0010592937469482422 seconds +23: +19: Time to load utils op: 0.0009613037109375 seconds +22: Time to load utils op: 0.0005471706390380859 seconds +22: Time to load utils op: 0.0007009506225585938 seconds +22: Time to load utils op: 0.0009033679962158203 seconds +22: Time to load utils op: 0.0008265972137451172 seconds +22: Time to load utils op: 0.0006704330444335938 seconds +22: Time to load utils op: 0.0007781982421875 seconds +22: Time to load utils op: 0.0007429122924804688 seconds +22: Time to load utils op: 0.0008978843688964844 seconds + 0: [2023-05-13 09:38:24,888] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 + 0: [2023-05-13 09:38:24,888] [INFO] [utils.py:828:see_memory_usage] MA 8.04 GB Max_MA 8.04 GB CA 10.11 GB Max_CA 10 GB + 0: [2023-05-13 09:38:24,888] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.9 GB, percent = 8.3% +29: Time to load utils op: 0.0005879402160644531 seconds +29: Time to load utils op: 0.0005669593811035156 seconds +29: Time to load utils op: 0.0005118846893310547 secondsTime to load utils op: 0.0004944801330566406 secondsTime to load utils op: 0.0004918575286865234 seconds +29: Time to load utils op: 0.00048232078552246094 secondsTime to load utils op: 0.000484466552734375 seconds +29: +29: +29: +29: Time to load utils op: 0.0004968643188476562 seconds +13: Time to load utils op: 0.0005288124084472656 secondsTime to load utils op: 0.0003800392150878906 seconds +13: +13: Time to load utils op: 0.0005700588226318359 seconds +13: Time to load utils op: 0.0005097389221191406 seconds +13: Time to load utils op: 0.0006868839263916016 secondsTime to load utils op: 0.0006356239318847656 secondsTime to load utils op: 0.000640869140625 secondsTime to load utils op: 0.0006399154663085938 seconds +13: +13: +13: +10: Time to load utils op: 0.0012514591217041016 seconds +10: Time to load utils op: 0.0012164115905761719 seconds +10: Time to load utils op: 0.001466512680053711 seconds +10: Time to load utils op: 0.0014951229095458984 seconds +10: Time to load utils op: 0.0014755725860595703 secondsTime to load utils op: 0.0015680789947509766 seconds +10: +10: Time to load utils op: 0.0015077590942382812 seconds +10: Time to load utils op: 0.001455545425415039 seconds +26: Time to load utils op: 0.0006926059722900391 seconds +26: Time to load utils op: 0.0005910396575927734 secondsTime to load utils op: 0.0007188320159912109 seconds +26: +26: Time to load utils op: 0.0007312297821044922 seconds +20: Time to load utils op: 0.0007226467132568359 seconds +26: Time to load utils op: 0.0007822513580322266 secondsTime to load utils op: 0.0007565021514892578 seconds +26: +26: Time to load utils op: 0.0007419586181640625 secondsTime to load utils op: 0.0007112026214599609 seconds +26: +20: Time to load utils op: 0.00075531005859375 seconds +20: Time to load utils op: 0.0008013248443603516 seconds +20: Time to load utils op: 0.0007815361022949219 seconds +20: Time to load utils op: 0.0007941722869873047 seconds +20: Time to load utils op: 0.0008220672607421875 seconds +20: Time to load utils op: 0.0007920265197753906 seconds +20: Time to load utils op: 0.0008444786071777344 seconds +18: Time to load utils op: 0.0009555816650390625 seconds +18: Time to load utils op: 0.001112222671508789 seconds +18: Time to load utils op: 0.0010457038879394531 seconds +18: Time to load utils op: 0.0011088848114013672 seconds +18: Time to load utils op: 0.001026153564453125 seconds +18: Time to load utils op: 0.0011012554168701172 seconds +18: Time to load utils op: 0.0010564327239990234 seconds +18: Time to load utils op: 0.0010819435119628906 seconds +24: Time to load utils op: 0.0008754730224609375 secondsTime to load utils op: 0.0007529258728027344 secondsTime to load utils op: 0.0008223056793212891 seconds +24: +24: +24: Time to load utils op: 0.0007481575012207031 seconds +24: Time to load utils op: 0.000797271728515625 seconds +24: Time to load utils op: 0.0009107589721679688 seconds +24: Time to load utils op: 0.0007884502410888672 seconds +24: Time to load utils op: 0.0009062290191650391 seconds +25: Time to load utils op: 0.0016274452209472656 seconds +25: Time to load utils op: 0.0016498565673828125 seconds +25: Time to load utils op: 0.001913309097290039 seconds +25: Time to load utils op: 0.0019385814666748047 seconds +25: Time to load utils op: 0.0020780563354492188 seconds +25: Time to load utils op: 0.001943349838256836 secondsTime to load utils op: 0.001970052719116211 seconds +25: +25: Time to load utils op: 0.001966238021850586 seconds +30: Time to load utils op: 0.0005156993865966797 seconds +30: Time to load utils op: 0.0004150867462158203 secondsTime to load utils op: 0.00042247772216796875 seconds +30: +30: Time to load utils op: 0.0005054473876953125 secondsTime to load utils op: 0.0004699230194091797 secondsTime to load utils op: 0.0004916191101074219 secondsTime to load utils op: 0.0004868507385253906 seconds +30: +30: +30: +30: Time to load utils op: 0.0004429817199707031 seconds + 1: Time to load utils op: 0.0008084774017333984 seconds + 1: Time to load utils op: 0.0007195472717285156 seconds + 1: Time to load utils op: 0.0007762908935546875 seconds + 1: Time to load utils op: 0.0009968280792236328 seconds + 1: Time to load utils op: 0.0007772445678710938 secondsTime to load utils op: 0.0009112358093261719 seconds + 1: Time to load utils op: 0.00080108642578125 seconds + 1: + 1: Time to load utils op: 0.0008795261383056641 seconds + 0: [2023-05-13 09:38:24,991] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 + 0: [2023-05-13 09:38:24,991] [INFO] [utils.py:828:see_memory_usage] MA 11.92 GB Max_MA 11.92 GB CA 15.91 GB Max_CA 16 GB + 0: [2023-05-13 09:38:24,992] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.9 GB, percent = 8.3% +17: Time to load utils op: 0.0007684230804443359 seconds +17: Time to load utils op: 0.0008442401885986328 secondsTime to load utils op: 0.0008218288421630859 seconds +17: +12: Time to load utils op: 0.0007152557373046875 seconds +12: Time to load utils op: 0.0007658004760742188 seconds +17: Time to load utils op: 0.0007560253143310547 seconds + 2: Time to load utils op: 0.0006868839263916016 seconds + 9: Time to load utils op: 0.0008215904235839844 seconds + 9: Time to load utils op: 0.0007581710815429688 secondsTime to load utils op: 0.0007376670837402344 seconds + 9: + 2: Time to load utils op: 0.0008034706115722656 seconds +12: Time to load utils op: 0.0009047985076904297 seconds +17: Time to load utils op: 0.0009644031524658203 seconds + 9: Time to load utils op: 0.0006971359252929688 seconds + 2: Time to load utils op: 0.0007767677307128906 seconds +12: Time to load utils op: 0.0008068084716796875 seconds +12: Time to load utils op: 0.0008289813995361328 seconds +17: Time to load utils op: 0.0009617805480957031 seconds +17: Time to load utils op: 0.0010304450988769531 secondsTime to load utils op: 0.0009908676147460938 seconds +17: + 9: Time to load utils op: 0.0008237361907958984 seconds + 9: Time to load utils op: 0.0007822513580322266 secondsTime to load utils op: 0.0007398128509521484 seconds + 9: + 2: Time to load utils op: 0.0009028911590576172 secondsTime to load utils op: 0.0009598731994628906 seconds + 2: +12: Time to load utils op: 0.0010249614715576172 seconds +12: Time to load utils op: 0.0008738040924072266 seconds +12: Time to load utils op: 0.0009174346923828125 seconds + 9: Time to load utils op: 0.0008451938629150391 seconds + 2: Time to load utils op: 0.0011067390441894531 secondsTime to load utils op: 0.0010268688201904297 seconds + 2: + 2: Time to load utils op: 0.0008604526519775391 seconds + 0: [2023-05-13 09:38:25,090] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 + 0: [2023-05-13 09:38:25,090] [INFO] [utils.py:828:see_memory_usage] MA 11.92 GB Max_MA 11.92 GB CA 15.91 GB Max_CA 16 GB + 0: [2023-05-13 09:38:25,091] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.9 GB, percent = 8.3% + 0: [2023-05-13 09:38:25,193] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 + 0: [2023-05-13 09:38:25,194] [INFO] [utils.py:828:see_memory_usage] MA 11.93 GB Max_MA 11.93 GB CA 15.91 GB Max_CA 16 GB + 0: [2023-05-13 09:38:25,194] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.9 GB, percent = 8.3% + 0: [2023-05-13 09:38:25,291] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer + 0: [2023-05-13 09:38:25,292] [INFO] [utils.py:828:see_memory_usage] MA 11.93 GB Max_MA 11.93 GB CA 15.91 GB Max_CA 16 GB + 0: [2023-05-13 09:38:25,292] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.9 GB, percent = 8.3% + 0: [2023-05-13 09:38:25,395] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer + 0: [2023-05-13 09:38:25,396] [INFO] [utils.py:828:see_memory_usage] MA 12.05 GB Max_MA 12.05 GB CA 15.91 GB Max_CA 16 GB + 0: [2023-05-13 09:38:25,396] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.9 GB, percent = 8.3% + 0: [2023-05-13 09:38:25,536] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer + 0: [2023-05-13 09:38:25,542] [INFO] [utils.py:828:see_memory_usage] MA 12.05 GB Max_MA 12.05 GB CA 15.91 GB Max_CA 16 GB + 0: [2023-05-13 09:38:25,544] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 41.9 GB, percent = 8.3% + 0: [2023-05-13 09:38:25,545] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam + 0: [2023-05-13 09:38:25,546] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler + 0: [2023-05-13 09:38:25,546] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = + 0: [2023-05-13 09:38:25,547] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] + 0: [2023-05-13 09:38:25,551] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: + 0: [2023-05-13 09:38:25,554] [INFO] [config.py:1011:print] activation_checkpointing_config { + 0: "partition_activations": false, + 0: "contiguous_memory_optimization": false, + 0: "cpu_checkpointing": false, + 0: "number_checkpoints": null, + 0: "synchronize_checkpoint_boundary": false, + 0: "profile": false + 0: } + 0: [2023-05-13 09:38:25,554] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} + 0: [2023-05-13 09:38:25,554] [INFO] [config.py:1011:print] amp_enabled .................. False + 0: [2023-05-13 09:38:25,554] [INFO] [config.py:1011:print] amp_params ................... False + 0: [2023-05-13 09:38:25,556] [INFO] [config.py:1011:print] autotuning_config ............ { + 0: "enabled": false, + 0: "start_step": null, + 0: "end_step": null, + 0: "metric_path": null, + 0: "arg_mappings": null, + 0: "metric": "throughput", + 0: "model_info": null, + 0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", + 0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", + 0: "overwrite": true, + 0: "fast": true, + 0: "start_profile_step": 3, + 0: "end_profile_step": 5, + 0: "tuner_type": "gridsearch", + 0: "tuner_early_stopping": 5, + 0: "tuner_num_trials": 50, + 0: "model_info_path": null, + 0: "mp_size": 1, + 0: "max_train_batch_size": null, + 0: "min_train_batch_size": 1, + 0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, + 0: "min_train_micro_batch_size_per_gpu": 1, + 0: "num_tuning_micro_batch_sizes": 3 + 0: } + 0: [2023-05-13 09:38:25,556] [INFO] [config.py:1011:print] bfloat16_enabled ............. True + 0: [2023-05-13 09:38:25,556] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False + 0: [2023-05-13 09:38:25,557] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True + 0: [2023-05-13 09:38:25,557] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False + 0: [2023-05-13 09:38:25,557] [INFO] [config.py:1011:print] comms_config ................. + 0: [2023-05-13 09:38:25,557] [INFO] [config.py:1011:print] communication_data_type ...... None + 0: [2023-05-13 09:38:25,558] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa + 0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} + 0: [2023-05-13 09:38:25,558] [INFO] [config.py:1011:print] curriculum_enabled ........... False + 0: [2023-05-13 09:38:25,558] [INFO] [config.py:1011:print] curriculum_params ............ False + 0: [2023-05-13 09:38:25,558] [INFO] [config.py:1011:print] dataloader_drop_last ......... False + 0: [2023-05-13 09:38:25,558] [INFO] [config.py:1011:print] disable_allgather ............ False + 0: [2023-05-13 09:38:25,559] [INFO] [config.py:1011:print] dump_state ................... False + 0: [2023-05-13 09:38:25,559] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None + 0: [2023-05-13 09:38:25,559] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False + 0: [2023-05-13 09:38:25,559] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 + 0: [2023-05-13 09:38:25,559] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer + 0: [2023-05-13 09:38:25,559] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 + 0: [2023-05-13 09:38:25,560] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 + 0: [2023-05-13 09:38:25,560] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 + 0: [2023-05-13 09:38:25,560] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 + 0: [2023-05-13 09:38:25,560] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False + 0: [2023-05-13 09:38:25,560] [INFO] [config.py:1011:print] elasticity_enabled ........... False + 0: [2023-05-13 09:38:25,561] [INFO] [config.py:1011:print] flops_profiler_config ........ { + 0: "enabled": false, + 0: "profile_step": 1, + 0: "module_depth": -1, + 0: "top_modules": 1, + 0: "detailed": true, + 0: "output_file": null + 0: } + 0: [2023-05-13 09:38:25,561] [INFO] [config.py:1011:print] fp16_auto_cast ............... None + 0: [2023-05-13 09:38:25,561] [INFO] [config.py:1011:print] fp16_enabled ................. False + 0: [2023-05-13 09:38:25,561] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False + 0: [2023-05-13 09:38:25,562] [INFO] [config.py:1011:print] global_rank .................. 0 + 0: [2023-05-13 09:38:25,562] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 4 + 0: [2023-05-13 09:38:25,562] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 + 0: [2023-05-13 09:38:25,562] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 + 0: [2023-05-13 09:38:25,562] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 + 0: [2023-05-13 09:38:25,563] [INFO] [config.py:1011:print] load_universal_checkpoint .... False + 0: [2023-05-13 09:38:25,563] [INFO] [config.py:1011:print] loss_scale ................... 1.0 + 0: [2023-05-13 09:38:25,563] [INFO] [config.py:1011:print] memory_breakdown ............. False + 0: [2023-05-13 09:38:25,563] [INFO] [config.py:1011:print] monitor_config ............... + 0: [2023-05-13 09:38:25,564] [INFO] [config.py:1011:print] nebula_config ................ { + 0: "enabled": false, + 0: "persistent_storage_path": null, + 0: "persistent_time_interval": 100, + 0: "num_of_version_in_retention": 2, + 0: "enable_nebula_load": true, + 0: "load_path": null + 0: } + 0: [2023-05-13 09:38:25,564] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False + 0: [2023-05-13 09:38:25,564] [INFO] [config.py:1011:print] optimizer_name ............... None + 0: [2023-05-13 09:38:25,564] [INFO] [config.py:1011:print] optimizer_params ............. None + 0: [2023-05-13 09:38:25,565] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} + 0: [2023-05-13 09:38:25,565] [INFO] [config.py:1011:print] pld_enabled .................. False + 0: [2023-05-13 09:38:25,565] [INFO] [config.py:1011:print] pld_params ................... False + 0: [2023-05-13 09:38:25,565] [INFO] [config.py:1011:print] prescale_gradients ........... False + 0: [2023-05-13 09:38:25,565] [INFO] [config.py:1011:print] scheduler_name ............... None + 0: [2023-05-13 09:38:25,565] [INFO] [config.py:1011:print] scheduler_params ............. None + 0: [2023-05-13 09:38:25,566] [INFO] [config.py:1011:print] sparse_attention ............. None + 0: [2023-05-13 09:38:25,566] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False + 0: [2023-05-13 09:38:25,566] [INFO] [config.py:1011:print] steps_per_print .............. 2000 + 0: [2023-05-13 09:38:25,566] [INFO] [config.py:1011:print] train_batch_size ............. 1024 + 0: [2023-05-13 09:38:25,566] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 2 + 0: [2023-05-13 09:38:25,567] [INFO] [config.py:1011:print] use_node_local_storage ....... False + 0: [2023-05-13 09:38:25,567] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False + 0: [2023-05-13 09:38:25,567] [INFO] [config.py:1011:print] world_size ................... 128 + 0: [2023-05-13 09:38:25,567] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False + 0: [2023-05-13 09:38:25,568] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False + 0: [2023-05-13 09:38:25,568] [INFO] [config.py:1011:print] zero_enabled ................. False + 0: [2023-05-13 09:38:25,568] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 + 0: [2023-05-13 09:38:25,568] [INFO] [config.py:996:print_user_config] json = { + 0: "train_micro_batch_size_per_gpu": 2, + 0: "train_batch_size": 1.024000e+03, + 0: "gradient_clipping": 1.0, + 0: "zero_optimization": { + 0: "stage": 0 + 0: }, + 0: "bf16": { + 0: "enabled": true + 0: }, + 0: "steps_per_print": 2.000000e+03, + 0: "wall_clock_breakdown": false + 0: } + 0: Time to load utils op: 0.0023877620697021484 seconds + 0: [2023-05-13 09:38:25,572] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=4 micro_batch_size=2 + 0: [2023-05-13 09:38:25,729] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=43 [0, 43) STAGE_PARAMS=2123243520 (2123.244M) TOTAL_PARAMS=4246487040 (4246.487M) UNIQUE_PARAMS=4246487040 (4246.487M) + 0: [2023-05-13 09:38:25,729] [INFO] [engine.py:145:__init__] RANK=1 STAGE=0 LAYERS=43 [0, 43) STAGE_PARAMS=2123243520 (2123.244M) TOTAL_PARAMS=4246487040 (4246.487M) UNIQUE_PARAMS=4246487040 (4246.487M) +16: [2023-05-13 09:38:28,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 4: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 4: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 2: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +12: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 2: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +26: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 4: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +19: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 6: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +26: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +27: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +20: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +17: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 7: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +25: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +14: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +27: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 7: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 7: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 7: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +20: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +29: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 1: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +25: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +23: [2023-05-13 09:38:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +14: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +11: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +10: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +10: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +10: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +16: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +13: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +16: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 8: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 8: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +21: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +13: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +21: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +21: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +20: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 8: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +26: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +20: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +20: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 8: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +20: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +21: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +21: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +14: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +14: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +24: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +11: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +29: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +26: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 9: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 9: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 9: [2023-05-13 09:38:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 9: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 9: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +24: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +14: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +14: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +27: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +15: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +29: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 0: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +24: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +23: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +23: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 7: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 7: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +29: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +29: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 9: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +29: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +30: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +23: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +13: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +15: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +27: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +15: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +27: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. + 0: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:28,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:28,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:28,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:28,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +26: [2023-05-13 09:38:28,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +26: [2023-05-13 09:38:28,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:28,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +16: [2023-05-13 09:38:28,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:28,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:28,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:28,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:28,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:28,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:28,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +13: [2023-05-13 09:38:28,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 5: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:28,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:28,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:28,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +13: [2023-05-13 09:38:28,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +13: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 5: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 5: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +13: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 5: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 5: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +19: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:28,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:28,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:28,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:28,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:28,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:28,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +19: [2023-05-13 09:38:28,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:28,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:28,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:28,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +19: [2023-05-13 09:38:28,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +19: [2023-05-13 09:38:28,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +19: [2023-05-13 09:38:28,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:28,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:28,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +23: [2023-05-13 09:38:28,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:28,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:28,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:28,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:28,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:28,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:28,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:28,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:28,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:28,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:28,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:28,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:28,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +18: [2023-05-13 09:38:28,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:28,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:28,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:28,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:28,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:28,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:28,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:28,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:28,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:28,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:28,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:28,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:28,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:28,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:28,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:28,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:28,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:28,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:28,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:28,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:28,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:28,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 8: [2023-05-13 09:38:28,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 8: [2023-05-13 09:38:28,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 8: [2023-05-13 09:38:28,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 0: [2023-05-13 09:38:28,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 0: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 0: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 0: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:28,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 1: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:28,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:28,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:28,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 9: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +10: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +10: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +31: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +17: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +16: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 9: [2023-05-13 09:38:28,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:28,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:28,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 3: [2023-05-13 09:38:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:28,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 3: [2023-05-13 09:38:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 3: [2023-05-13 09:38:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +30: [2023-05-13 09:38:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +30: [2023-05-13 09:38:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:28,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 1: [2023-05-13 09:38:28,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:28,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 9: [2023-05-13 09:38:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 3: [2023-05-13 09:38:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +23: [2023-05-13 09:38:28,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 1: [2023-05-13 09:38:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:28,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +17: [2023-05-13 09:38:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +17: [2023-05-13 09:38:28,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:28,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:28,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 1: [2023-05-13 09:38:28,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:28,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:28,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:28,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:28,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:28,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:28,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:28,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:28,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:28,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:28,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:28,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:28,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:28,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:28,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:28,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:28,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:28,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:28,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:28,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:28,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:28,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:28,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:28,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:28,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:28,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:28,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:28,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:28,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:28,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:28,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:28,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:28,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:28,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:28,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:28,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:28,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:28,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:28,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:28,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:28,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:28,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:28,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:28,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:28,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:28,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:28,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:28,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:28,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:28,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:28,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:28,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:28,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:28,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:28,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:28,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:28,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:28,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:28,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:28,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:28,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:28,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:28,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:28,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:28,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:28,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:28,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:28,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:28,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:29,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +26: [2023-05-13 09:38:29,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +26: [2023-05-13 09:38:29,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +26: [2023-05-13 09:38:29,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +26: [2023-05-13 09:38:29,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +20: [2023-05-13 09:38:29,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +20: [2023-05-13 09:38:29,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:29,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:29,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:29,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:29,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +16: [2023-05-13 09:38:29,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +20: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +20: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +20: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +18: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +18: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +18: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +16: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +16: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +27: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +27: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +24: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +24: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +24: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +20: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +27: [2023-05-13 09:38:29,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +18: [2023-05-13 09:38:29,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +23: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +23: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +23: [2023-05-13 09:38:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +27: [2023-05-13 09:38:29,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +23: [2023-05-13 09:38:29,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:29,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +16: [2023-05-13 09:38:29,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:29,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +14: [2023-05-13 09:38:29,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +14: [2023-05-13 09:38:29,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +14: [2023-05-13 09:38:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +22: [2023-05-13 09:38:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +22: [2023-05-13 09:38:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +22: [2023-05-13 09:38:29,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +11: [2023-05-13 09:38:29,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +11: [2023-05-13 09:38:29,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +10: [2023-05-13 09:38:29,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:29,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +13: [2023-05-13 09:38:29,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +13: [2023-05-13 09:38:29,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +13: [2023-05-13 09:38:29,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:29,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:29,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:29,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:29,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:29,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:29,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:29,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +24: [2023-05-13 09:38:29,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:29,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:29,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:29,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +28: [2023-05-13 09:38:29,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +28: [2023-05-13 09:38:29,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:29,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:29,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:29,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:29,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:29,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:29,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:29,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:29,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +22: [2023-05-13 09:38:29,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:29,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:29,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:29,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +25: [2023-05-13 09:38:29,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:29,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +26: [2023-05-13 09:38:29,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +28: [2023-05-13 09:38:29,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +17: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +17: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +17: [2023-05-13 09:38:29,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:29,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:29,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:29,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 3: [2023-05-13 09:38:29,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 3: [2023-05-13 09:38:29,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 3: [2023-05-13 09:38:29,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 3: [2023-05-13 09:38:29,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +29: [2023-05-13 09:38:29,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +29: [2023-05-13 09:38:29,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +29: [2023-05-13 09:38:29,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +29: [2023-05-13 09:38:29,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:29,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 9: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +12: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +12: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +12: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 9: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 9: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +12: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 0: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 0: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 0: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +24: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:29,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +31: [2023-05-13 09:38:29,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +31: [2023-05-13 09:38:29,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 7: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 1: [2023-05-13 09:38:29,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 1: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +21: [2023-05-13 09:38:29,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:29,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:29,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +29: [2023-05-13 09:38:29,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:29,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +29: [2023-05-13 09:38:29,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 6: [2023-05-13 09:38:29,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:29,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +14: [2023-05-13 09:38:29,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +14: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +29: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +14: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +21: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 6: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +24: [2023-05-13 09:38:29,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +30: [2023-05-13 09:38:29,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +30: [2023-05-13 09:38:29,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:29,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:29,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +23: [2023-05-13 09:38:29,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:29,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:29,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +12: [2023-05-13 09:38:29,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:29,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 6: [2023-05-13 09:38:29,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +14: [2023-05-13 09:38:29,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +11: [2023-05-13 09:38:29,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +10: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +10: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +28: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_01-model_01-model_states.pt. +16: [2023-05-13 09:38:29,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:29,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:29,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +30: [2023-05-13 09:38:29,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:29,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:29,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:29,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:29,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:29,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:29,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:29,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:29,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +11: [2023-05-13 09:38:29,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +21: [2023-05-13 09:38:29,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:29,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +30: [2023-05-13 09:38:29,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:29,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:29,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +28: [2023-05-13 09:38:29,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:29,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:29,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +11: [2023-05-13 09:38:29,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +11: [2023-05-13 09:38:29,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:29,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:29,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:29,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:29,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:29,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:29,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:29,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +18: [2023-05-13 09:38:29,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:29,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +18: [2023-05-13 09:38:29,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +18: [2023-05-13 09:38:29,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:29,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:29,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:29,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:29,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +17: [2023-05-13 09:38:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +17: [2023-05-13 09:38:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:29,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:29,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:29,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:29,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:29,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 1: [2023-05-13 09:38:29,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +11: [2023-05-13 09:38:29,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:29,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:29,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 1: [2023-05-13 09:38:29,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +17: [2023-05-13 09:38:29,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:29,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:29,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:29,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +17: [2023-05-13 09:38:29,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +25: [2023-05-13 09:38:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:29,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:29,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:29,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +25: [2023-05-13 09:38:29,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +25: [2023-05-13 09:38:29,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +25: [2023-05-13 09:38:29,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +11: [2023-05-13 09:38:29,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:29,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +25: [2023-05-13 09:38:29,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:29,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +12: [2023-05-13 09:38:29,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +12: [2023-05-13 09:38:29,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +29: [2023-05-13 09:38:29,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +12: [2023-05-13 09:38:29,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:29,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:29,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:29,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +29: [2023-05-13 09:38:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +29: [2023-05-13 09:38:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +29: [2023-05-13 09:38:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 6: [2023-05-13 09:38:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +12: [2023-05-13 09:38:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:29,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:29,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +10: [2023-05-13 09:38:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +17: [2023-05-13 09:38:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:29,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +10: [2023-05-13 09:38:29,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:29,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +29: [2023-05-13 09:38:29,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:29,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:29,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:29,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:29,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:29,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:29,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:29,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:29,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:29,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:29,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:29,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:29,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:29,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:29,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:29,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:29,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:29,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:29,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:29,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:29,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:29,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:29,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:29,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:29,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:29,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:29,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:29,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:29,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:29,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:29,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:29,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:29,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:29,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:29,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:29,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:29,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:29,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:29,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:29,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:29,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:29,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:29,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:29,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:29,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +11: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +11: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +11: [2023-05-13 09:38:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:29,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:29,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:29,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:29,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +19: [2023-05-13 09:38:29,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:29,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:29,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:29,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:29,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 5: [2023-05-13 09:38:29,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:29,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:29,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:29,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:29,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:29,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:29,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:29,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:29,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:29,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:29,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:29,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:29,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +26: [2023-05-13 09:38:29,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:29,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +29: [2023-05-13 09:38:29,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +20: [2023-05-13 09:38:29,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +20: [2023-05-13 09:38:29,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:29,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:29,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:29,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:29,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:29,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:29,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +14: [2023-05-13 09:38:29,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:29,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:29,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:29,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:29,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:29,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:29,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:29,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:29,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:29,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:29,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:29,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:29,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:29,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:29,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:29,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:29,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:29,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:29,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:29,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:29,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:29,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:29,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:29,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:29,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:29,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:29,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:29,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:29,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +31: [2023-05-13 09:38:29,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +13: [2023-05-13 09:38:29,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:29,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:29,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:29,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:29,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:29,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:29,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:29,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +31: [2023-05-13 09:38:29,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +31: [2023-05-13 09:38:29,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +13: [2023-05-13 09:38:29,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:29,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:29,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:29,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:29,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:29,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +31: [2023-05-13 09:38:29,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:29,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +27: [2023-05-13 09:38:29,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +16: [2023-05-13 09:38:29,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +16: [2023-05-13 09:38:29,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:29,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +27: [2023-05-13 09:38:29,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +27: [2023-05-13 09:38:29,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +27: [2023-05-13 09:38:29,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:29,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:29,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +26: [2023-05-13 09:38:29,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 9: [2023-05-13 09:38:29,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:29,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +26: [2023-05-13 09:38:29,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:29,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +25: [2023-05-13 09:38:29,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +27: [2023-05-13 09:38:29,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +25: [2023-05-13 09:38:29,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +25: [2023-05-13 09:38:29,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +20: [2023-05-13 09:38:29,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:29,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:29,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +26: [2023-05-13 09:38:29,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +14: [2023-05-13 09:38:29,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +27: [2023-05-13 09:38:29,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:29,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:29,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:29,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:29,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:29,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:29,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:29,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:29,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:29,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:29,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:29,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:29,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 2: [2023-05-13 09:38:29,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:29,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:29,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:29,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:29,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:29,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:29,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:29,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:29,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:29,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:29,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:29,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:29,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:29,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:29,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:29,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:29,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:29,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 4: [2023-05-13 09:38:29,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:29,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:29,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:29,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:29,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:29,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:29,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:29,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:29,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:29,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:29,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:29,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:29,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:29,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:29,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:29,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:29,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:29,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:29,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +21: [2023-05-13 09:38:29,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +21: [2023-05-13 09:38:29,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +31: [2023-05-13 09:38:29,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:29,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +12: [2023-05-13 09:38:29,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +12: [2023-05-13 09:38:29,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +31: [2023-05-13 09:38:29,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:29,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:29,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:29,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +20: [2023-05-13 09:38:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 1: [2023-05-13 09:38:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:29,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:29,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:29,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:29,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:29,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:29,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:29,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:29,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:29,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +18: [2023-05-13 09:38:29,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:29,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +14: [2023-05-13 09:38:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 6: [2023-05-13 09:38:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:29,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:29,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:29,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:29,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:29,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +13: [2023-05-13 09:38:29,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +23: [2023-05-13 09:38:29,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +14: [2023-05-13 09:38:29,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:29,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:29,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:29,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:29,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +23: [2023-05-13 09:38:29,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:29,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:29,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:29,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +14: [2023-05-13 09:38:29,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +18: [2023-05-13 09:38:29,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:29,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:29,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:29,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +23: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +23: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +25: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 6: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 6: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +13: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:30,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:30,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +25: [2023-05-13 09:38:30,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:29,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 6: [2023-05-13 09:38:30,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:30,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 3: [2023-05-13 09:38:30,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +25: [2023-05-13 09:38:30,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:30,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +13: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +21: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +29: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +29: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +29: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +31: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +31: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +25: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:30,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:30,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +25: [2023-05-13 09:38:30,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +13: [2023-05-13 09:38:30,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +25: [2023-05-13 09:38:30,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +13: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:30,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +13: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:30,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:30,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +31: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:30,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:30,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:30,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +27: [2023-05-13 09:38:30,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:30,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +31: [2023-05-13 09:38:30,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +27: [2023-05-13 09:38:30,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:30,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:30,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:30,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:30,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:30,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:30,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:30,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:30,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:30,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +11: [2023-05-13 09:38:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +11: [2023-05-13 09:38:30,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:30,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:30,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +11: [2023-05-13 09:38:30,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +11: [2023-05-13 09:38:30,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:30,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:30,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:30,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:30,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +24: [2023-05-13 09:38:30,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +20: [2023-05-13 09:38:30,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:30,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:30,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 2: [2023-05-13 09:38:30,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 2: [2023-05-13 09:38:30,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 1: [2023-05-13 09:38:30,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:30,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:30,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +18: [2023-05-13 09:38:30,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:30,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +29: [2023-05-13 09:38:30,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:30,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:30,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:30,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:30,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:30,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +29: [2023-05-13 09:38:30,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:30,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:30,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:30,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +19: [2023-05-13 09:38:30,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +20: [2023-05-13 09:38:30,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 7: [2023-05-13 09:38:30,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:30,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:30,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:30,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +19: [2023-05-13 09:38:30,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:30,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 7: [2023-05-13 09:38:30,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:30,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:30,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +17: [2023-05-13 09:38:30,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +17: [2023-05-13 09:38:30,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:30,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 7: [2023-05-13 09:38:30,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +13: [2023-05-13 09:38:30,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:30,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:30,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:30,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:30,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:30,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:30,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:30,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:30,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:30,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 1: [2023-05-13 09:38:30,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:30,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:30,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:30,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:30,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:30,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:30,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:30,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:30,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +12: [2023-05-13 09:38:30,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:30,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:30,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:30,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:30,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:30,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:30,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:30,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:30,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:30,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:30,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +12: [2023-05-13 09:38:30,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:30,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:30,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:30,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:30,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:30,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:30,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:30,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:30,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:30,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:30,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:30,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:30,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:30,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:30,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:30,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:30,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:30,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +29: [2023-05-13 09:38:30,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +29: [2023-05-13 09:38:30,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +29: [2023-05-13 09:38:30,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +21: [2023-05-13 09:38:30,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 1: [2023-05-13 09:38:30,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 1: [2023-05-13 09:38:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 1: [2023-05-13 09:38:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +11: [2023-05-13 09:38:30,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:30,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:30,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:30,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +11: [2023-05-13 09:38:30,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:30,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +24: [2023-05-13 09:38:30,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:30,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:30,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +11: [2023-05-13 09:38:30,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:30,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:30,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:30,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:30,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:30,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:30,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:30,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:30,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:30,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +24: [2023-05-13 09:38:30,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:30,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:30,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:30,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:30,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:30,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:30,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:30,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:30,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:30,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:30,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:30,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:30,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:30,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:30,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:30,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:30,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +15: [2023-05-13 09:38:30,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:30,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:30,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:30,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 4: [2023-05-13 09:38:30,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 4: [2023-05-13 09:38:30,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:30,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_03-model_01-model_states.pt. + 3: [2023-05-13 09:38:30,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:30,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:30,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:30,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:30,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:30,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:30,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 1: [2023-05-13 09:38:30,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 1: [2023-05-13 09:38:30,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:30,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:30,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:30,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:30,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 1: [2023-05-13 09:38:30,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 4: [2023-05-13 09:38:30,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:30,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 0: [2023-05-13 09:38:30,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:30,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:30,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:30,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:30,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:30,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:30,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:30,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:30,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 0: [2023-05-13 09:38:30,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 0: [2023-05-13 09:38:30,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:30,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:30,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... + 4: [2023-05-13 09:38:30,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:30,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:30,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:30,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:30,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:30,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:30,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:30,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:30,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:30,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:30,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:30,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:30,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:30,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:30,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:30,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:30,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:30,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:30,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:30,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:30,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:30,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:30,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:30,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:30,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:30,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:30,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:30,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:30,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:30,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:30,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:30,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:30,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:30,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:30,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:30,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:30,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:30,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:30,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:30,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:30,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:30,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:30,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:30,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:30,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:30,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:30,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:30,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:30,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:30,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:30,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:30,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:30,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:30,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:30,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:30,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:30,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:30,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:30,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:30,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:30,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:30,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:30,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:30,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:30,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:30,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:30,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:30,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:30,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:30,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:30,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:30,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:30,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:30,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:30,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:30,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:30,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:30,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:30,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:30,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:30,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:30,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:30,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:30,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:30,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:30,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:30,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:30,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:30,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:30,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:30,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:30,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:30,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:30,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:30,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:30,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:30,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:30,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:30,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:30,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:30,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:30,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:30,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:30,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:30,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:30,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:30,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:30,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:30,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:30,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:30,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:30,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:30,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:30,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:30,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:30,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:30,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:30,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:30,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:30,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:30,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:30,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:30,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:30,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:30,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:30,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:30,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:30,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:30,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:30,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:30,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:30,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:30,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:30,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:30,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:30,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:30,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:30,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:30,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:30,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:30,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:30,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:30,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:30,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:30,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:30,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:30,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:30,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:30,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:30,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:30,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:30,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:30,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:30,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:30,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:30,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:30,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:30,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:30,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:30,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:30,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:30,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:30,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:30,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:30,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:30,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:30,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:30,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:30,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:30,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:30,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:30,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:30,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:30,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:30,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:30,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:30,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:30,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:30,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:30,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:30,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:30,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:30,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:30,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:30,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:30,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:30,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:30,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:30,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:30,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:30,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:30,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:30,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:30,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:30,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:30,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:30,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:30,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:30,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:30,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:30,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:30,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:30,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:30,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:30,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:30,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:30,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:30,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:30,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:30,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:30,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:30,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:30,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:30,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:30,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:30,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:30,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:30,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:30,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:30,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:30,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:30,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:30,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:30,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:30,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:30,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:30,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:30,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:30,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:30,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:30,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:30,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:30,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:30,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:30,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:30,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:30,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:30,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:30,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:30,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:30,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:30,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:30,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:30,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:30,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:30,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:30,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:30,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:30,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:30,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:30,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:30,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:30,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:30,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:30,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:30,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:30,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:30,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:30,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:30,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:30,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:30,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:30,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:30,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:30,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:30,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:30,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:30,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:30,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:30,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:30,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:30,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:30,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:30,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:30,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:30,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:30,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:30,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:30,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:30,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:30,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:30,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:30,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:30,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:30,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:30,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:30,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:30,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:30,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:30,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:30,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:30,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:30,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:30,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:30,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:30,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:30,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:30,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:30,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:30,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:30,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:30,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:30,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:30,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:30,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:30,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:30,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:30,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:30,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:30,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:30,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:30,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:30,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:30,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:30,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:30,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:30,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:30,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:30,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:30,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:30,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:30,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:30,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:30,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:30,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:30,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:30,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:30,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:30,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:30,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:30,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:30,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:30,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:30,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:30,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:30,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:30,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:30,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:30,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:30,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:30,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:30,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:30,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:30,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:30,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:30,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:30,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:30,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:30,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:30,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:30,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:30,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:30,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:30,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:30,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:30,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:30,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:30,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:30,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:30,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:30,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:30,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:30,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:30,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:30,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:30,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:30,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:30,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:30,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:30,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:30,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:30,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:30,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:30,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:30,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:30,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:30,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:30,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:30,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:30,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:30,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:30,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:30,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:30,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:30,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:30,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:30,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:30,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:30,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:30,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:30,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:30,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:30,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:30,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:30,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:30,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:30,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:30,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:30,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:30,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:30,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:30,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:30,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:30,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:30,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:30,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:30,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:30,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:30,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:30,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:30,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:30,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:30,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:30,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:30,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:30,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:30,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +23: [2023-05-13 09:38:30,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +30: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:30,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:30,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:30,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:30,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:30,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:30,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:30,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:30,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:30,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:30,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:30,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:30,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +30: [2023-05-13 09:38:30,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:30,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +26: [2023-05-13 09:38:30,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +23: [2023-05-13 09:38:30,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:30,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:30,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:30,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:30,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:30,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:30,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:30,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:30,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:30,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:30,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:30,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:30,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:30,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:30,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:30,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:30,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:30,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:30,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:30,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:30,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:30,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:30,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:30,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:30,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:30,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:30,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:30,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:30,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:30,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:30,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:30,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:30,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:30,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:30,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:30,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:30,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:30,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:30,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:30,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:30,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:30,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:30,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:30,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:30,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:30,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:30,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:30,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:30,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:30,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:30,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +23: [2023-05-13 09:38:30,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +23: [2023-05-13 09:38:30,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +23: [2023-05-13 09:38:30,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:30,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:30,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:30,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:30,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:30,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:30,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +29: [2023-05-13 09:38:30,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:30,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:30,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:30,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:30,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:30,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +26: [2023-05-13 09:38:30,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +26: [2023-05-13 09:38:30,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:30,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:30,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:30,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:30,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:30,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:30,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:30,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:30,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:30,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:30,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:30,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:30,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:30,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:30,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:30,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:30,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:30,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:30,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:30,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:30,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +30: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +22: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 6: [2023-05-13 09:38:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:30,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:30,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:30,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:30,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:30,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:30,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:30,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:30,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:30,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:30,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:30,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:30,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:30,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:30,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:30,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:30,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:30,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:30,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:30,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:30,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:30,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:30,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:31,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:31,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +25: [2023-05-13 09:38:31,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +25: [2023-05-13 09:38:31,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +20: [2023-05-13 09:38:31,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +20: [2023-05-13 09:38:31,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +30: [2023-05-13 09:38:31,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:31,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:31,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +20: [2023-05-13 09:38:31,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +25: [2023-05-13 09:38:31,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +30: [2023-05-13 09:38:31,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:31,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:31,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:31,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:31,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:31,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:31,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:31,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:31,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:31,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +10: [2023-05-13 09:38:31,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:31,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +21: [2023-05-13 09:38:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +21: [2023-05-13 09:38:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +21: [2023-05-13 09:38:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +18: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +10: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +21: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:31,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +22: [2023-05-13 09:38:31,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:31,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:31,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:31,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:31,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:31,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:31,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:31,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:31,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:31,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:31,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:31,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:31,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:31,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:31,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:31,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:31,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:31,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:31,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:31,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:31,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:31,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:31,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:31,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:31,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:31,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:31,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:31,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:31,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:31,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:31,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:31,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:31,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:31,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:31,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:31,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:31,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:31,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:31,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:31,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:31,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:31,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:31,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:31,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:31,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:31,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:31,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:31,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:31,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:31,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:31,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:31,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:31,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:31,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:31,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:31,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +20: [2023-05-13 09:38:31,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:31,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +30: [2023-05-13 09:38:31,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:31,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:31,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:31,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:31,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:31,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:31,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:31,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:31,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:31,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:31,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:31,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:31,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:31,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:31,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:31,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:31,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:31,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:31,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:31,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:31,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:31,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:31,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:31,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:31,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +20: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:31,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:31,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:31,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:31,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:31,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:31,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:31,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:31,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:31,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:31,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 5: [2023-05-13 09:38:31,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 7: [2023-05-13 09:38:31,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 7: [2023-05-13 09:38:31,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:31,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:31,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 5: [2023-05-13 09:38:31,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 5: [2023-05-13 09:38:31,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 5: [2023-05-13 09:38:31,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 5: [2023-05-13 09:38:31,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:31,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:31,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:31,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:31,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:31,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:31,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:31,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:31,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:31,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:31,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:31,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:31,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:31,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:31,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:31,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:31,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:31,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:31,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +10: [2023-05-13 09:38:31,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:31,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:31,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:31,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:31,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:31,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:31,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:31,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:31,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:31,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:31,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:31,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:31,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:31,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:31,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:31,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:31,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:31,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:31,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +13: [2023-05-13 09:38:31,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +13: [2023-05-13 09:38:31,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +13: [2023-05-13 09:38:31,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:31,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:31,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:31,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:31,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +18: [2023-05-13 09:38:31,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +18: [2023-05-13 09:38:31,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +18: [2023-05-13 09:38:31,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:31,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:31,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:31,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:31,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:31,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:31,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:31,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:31,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:31,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:31,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:31,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:31,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:31,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:31,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:31,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:31,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:31,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:31,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:31,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:31,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:31,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:31,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:31,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:31,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:31,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:31,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:31,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:31,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:31,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +29: [2023-05-13 09:38:31,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:31,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +29: [2023-05-13 09:38:31,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:31,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:31,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:31,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:31,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:31,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:31,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:31,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:31,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:31,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:31,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:31,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:31,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:31,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +24: [2023-05-13 09:38:31,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:31,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:31,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:31,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:31,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +13: [2023-05-13 09:38:31,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:31,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:31,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:31,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:31,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +24: [2023-05-13 09:38:31,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +24: [2023-05-13 09:38:31,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:31,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +13: [2023-05-13 09:38:31,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:31,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 7: [2023-05-13 09:38:31,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +22: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +11: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +11: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +11: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +11: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +22: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +22: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:31,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:31,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:31,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:31,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:31,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:31,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:31,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:31,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:31,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:31,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:31,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:31,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:31,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:31,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:31,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:31,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:31,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:31,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:31,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:31,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:31,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:31,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:31,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:31,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:31,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:31,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:31,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 2: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 2: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +17: [2023-05-13 09:38:31,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:31,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +29: [2023-05-13 09:38:31,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +29: [2023-05-13 09:38:31,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:31,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +17: [2023-05-13 09:38:31,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:31,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:31,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:31,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:31,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +24: [2023-05-13 09:38:31,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:31,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:31,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:31,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:31,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:31,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:31,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:31,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +15: [2023-05-13 09:38:31,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:31,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +14: [2023-05-13 09:38:31,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +14: [2023-05-13 09:38:31,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:31,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:31,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:31,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +15: [2023-05-13 09:38:31,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:31,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:31,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:31,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:31,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:31,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:31,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:31,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:31,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:31,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:31,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:31,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:31,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:31,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:31,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:31,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:31,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:31,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:31,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:31,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:31,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:31,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:31,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:31,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:31,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +17: [2023-05-13 09:38:31,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:31,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:31,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:31,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +17: [2023-05-13 09:38:31,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:31,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:31,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +17: [2023-05-13 09:38:31,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:31,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:31,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:31,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:31,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:31,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:31,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:31,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:31,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:31,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:31,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:31,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:31,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:31,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:31,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:31,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:31,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:31,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:31,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:31,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:31,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:31,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:31,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:31,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:31,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:31,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:31,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:31,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:31,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:31,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:31,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:31,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:31,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +19: [2023-05-13 09:38:31,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:31,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:31,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:31,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:31,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:31,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:31,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:31,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:31,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:31,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:31,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:31,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:31,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:31,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +19: [2023-05-13 09:38:31,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +19: [2023-05-13 09:38:31,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +19: [2023-05-13 09:38:31,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:31,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:31,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:31,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +16: [2023-05-13 09:38:31,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +16: [2023-05-13 09:38:31,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +16: [2023-05-13 09:38:31,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:31,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:31,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +26: [2023-05-13 09:38:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +26: [2023-05-13 09:38:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +19: [2023-05-13 09:38:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:31,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:31,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:31,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:31,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:31,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:31,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:31,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:31,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:31,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:31,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:31,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:31,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:31,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:31,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:31,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:31,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:31,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:31,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:31,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +25: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +25: [2023-05-13 09:38:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +25: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +31: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:31,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:31,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:31,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:31,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:31,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:31,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +26: [2023-05-13 09:38:31,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:31,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:31,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:31,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:31,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:31,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:31,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:31,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:31,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:31,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:31,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:31,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:31,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:31,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 2: [2023-05-13 09:38:31,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:31,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:31,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:31,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:31,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:31,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:31,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:31,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:31,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:31,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:31,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:31,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +26: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:31,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:31,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +16: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +16: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +13: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:31,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:31,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:31,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:31,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:31,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:31,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:31,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:31,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:31,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:31,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:31,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:31,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:31,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:31,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:31,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +31: [2023-05-13 09:38:31,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:31,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:31,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:31,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +26: [2023-05-13 09:38:31,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:31,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:31,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +26: [2023-05-13 09:38:31,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +26: [2023-05-13 09:38:31,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +28: [2023-05-13 09:38:31,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:31,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 5: [2023-05-13 09:38:31,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:31,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:31,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:31,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:31,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:31,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:31,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:31,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:31,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:31,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:31,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:31,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +14: [2023-05-13 09:38:31,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:31,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:31,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:31,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:31,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +29: [2023-05-13 09:38:31,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:31,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:31,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +31: [2023-05-13 09:38:31,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:31,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:31,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:31,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:31,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:31,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:31,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:31,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:31,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:31,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:31,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:31,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:31,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:31,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +23: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:31,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:31,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:31,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:31,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:31,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:31,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:31,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +31: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +15: [2023-05-13 09:38:31,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:31,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:31,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:31,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:31,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:31,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:31,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 2: [2023-05-13 09:38:31,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:31,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:31,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 5: [2023-05-13 09:38:31,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:31,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:31,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:31,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:31,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:31,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:31,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:31,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:31,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:31,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +29: [2023-05-13 09:38:31,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:31,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:31,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:31,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +24: [2023-05-13 09:38:31,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +24: [2023-05-13 09:38:31,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +18: [2023-05-13 09:38:31,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:31,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:31,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:31,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:31,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:31,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:31,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:31,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:31,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:31,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:31,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:31,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:31,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +18: [2023-05-13 09:38:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:31,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:31,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:31,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 5: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 5: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:31,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:31,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:31,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +18: [2023-05-13 09:38:31,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 5: [2023-05-13 09:38:31,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +23: [2023-05-13 09:38:31,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +23: [2023-05-13 09:38:31,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:31,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:31,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:31,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:31,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +22: [2023-05-13 09:38:31,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:31,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:31,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:31,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:31,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:31,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:31,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:31,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:31,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:31,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:31,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:31,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:31,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:31,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:31,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:31,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:31,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:31,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:31,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:31,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:31,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:31,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:31,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:31,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:31,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:31,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:31,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:31,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:31,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:31,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:31,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:31,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:31,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:31,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:31,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:31,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:31,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:31,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:31,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:31,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +10: [2023-05-13 09:38:31,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +10: [2023-05-13 09:38:31,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +22: [2023-05-13 09:38:31,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:31,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +22: [2023-05-13 09:38:31,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:31,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:31,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:31,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:31,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:31,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:31,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:31,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:31,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:31,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:31,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:31,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:31,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:31,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +26: [2023-05-13 09:38:31,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:31,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:31,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:31,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:31,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:31,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:31,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:31,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:31,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:31,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:31,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:31,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:31,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:31,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:31,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:31,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:31,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:31,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:31,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:31,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:31,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:31,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:31,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:31,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:31,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:31,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:31,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:31,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:31,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:31,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:31,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:31,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:31,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:31,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:31,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:31,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:31,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:31,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:31,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:31,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:31,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:31,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:31,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:31,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:31,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:31,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:31,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:31,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:31,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:31,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 2: [2023-05-13 09:38:31,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +30: [2023-05-13 09:38:31,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:31,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:31,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:31,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +26: [2023-05-13 09:38:31,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:31,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:31,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +10: [2023-05-13 09:38:31,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +26: [2023-05-13 09:38:31,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:31,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:31,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:31,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:31,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:31,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:31,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:31,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:31,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:31,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:31,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:31,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:31,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:31,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +10: [2023-05-13 09:38:31,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +23: [2023-05-13 09:38:31,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:31,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:31,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:31,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:31,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:31,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:31,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:31,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:31,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:31,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:31,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:31,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:31,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:31,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:31,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:31,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +10: [2023-05-13 09:38:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:31,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:31,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:31,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +30: [2023-05-13 09:38:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +30: [2023-05-13 09:38:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +30: [2023-05-13 09:38:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:31,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:31,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +13: [2023-05-13 09:38:31,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:31,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:31,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:31,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:31,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:31,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:31,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:31,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:31,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:31,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:31,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:31,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:31,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:31,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:31,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:31,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:31,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +14: [2023-05-13 09:38:31,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:31,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:31,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:31,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:31,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:31,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:31,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:31,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:31,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:31,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:31,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:31,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:31,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:31,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:31,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:31,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +29: [2023-05-13 09:38:31,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +29: [2023-05-13 09:38:31,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +29: [2023-05-13 09:38:31,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:31,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:31,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:31,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:31,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +19: [2023-05-13 09:38:31,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:31,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:31,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:31,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +19: [2023-05-13 09:38:31,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:31,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +14: [2023-05-13 09:38:31,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +14: [2023-05-13 09:38:31,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +14: [2023-05-13 09:38:31,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:31,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:31,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:31,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:31,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:31,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:31,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:31,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +31: [2023-05-13 09:38:31,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +31: [2023-05-13 09:38:31,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +11: [2023-05-13 09:38:31,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +11: [2023-05-13 09:38:31,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +31: [2023-05-13 09:38:31,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +11: [2023-05-13 09:38:31,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:31,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:31,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:31,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:31,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:31,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:31,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 1: [2023-05-13 09:38:31,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:31,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +15: [2023-05-13 09:38:31,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +15: [2023-05-13 09:38:31,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +15: [2023-05-13 09:38:31,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +14: [2023-05-13 09:38:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:31,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:31,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:31,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:31,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +31: [2023-05-13 09:38:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +31: [2023-05-13 09:38:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:31,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:31,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:31,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:31,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:31,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:31,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:31,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:31,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:31,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:31,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt... +13: [2023-05-13 09:38:31,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:31,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:31,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:31,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:31,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:31,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:31,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:31,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:31,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:31,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +13: [2023-05-13 09:38:31,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +25: [2023-05-13 09:38:31,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +25: [2023-05-13 09:38:31,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +12: [2023-05-13 09:38:31,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:31,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +13: [2023-05-13 09:38:31,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:31,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:31,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:31,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +13: [2023-05-13 09:38:31,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:31,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:31,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:31,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +28: [2023-05-13 09:38:31,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:31,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:31,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:31,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +20: [2023-05-13 09:38:31,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +31: [2023-05-13 09:38:31,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:31,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +31: [2023-05-13 09:38:31,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:31,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:31,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +31: [2023-05-13 09:38:31,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:31,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:31,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:31,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +29: [2023-05-13 09:38:31,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +29: [2023-05-13 09:38:31,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +18: [2023-05-13 09:38:31,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +25: [2023-05-13 09:38:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:31,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:31,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +18: [2023-05-13 09:38:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:31,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +27: [2023-05-13 09:38:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +27: [2023-05-13 09:38:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +22: [2023-05-13 09:38:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +22: [2023-05-13 09:38:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +22: [2023-05-13 09:38:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +22: [2023-05-13 09:38:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:31,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:31,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:31,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +30: [2023-05-13 09:38:31,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:31,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:31,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:31,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +15: [2023-05-13 09:38:31,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +28: [2023-05-13 09:38:31,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:31,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +28: [2023-05-13 09:38:31,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:31,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:31,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:31,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:31,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:31,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:31,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:31,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:31,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +18: [2023-05-13 09:38:31,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:31,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +18: [2023-05-13 09:38:31,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +15: [2023-05-13 09:38:31,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:31,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:31,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +22: [2023-05-13 09:38:31,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:31,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:31,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:31,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:31,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:31,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:31,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:31,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:31,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:31,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:31,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:31,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:31,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:31,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +25: [2023-05-13 09:38:31,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +25: [2023-05-13 09:38:31,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:31,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:31,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:31,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:31,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:31,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:31,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:31,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:31,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:31,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:31,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:31,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:31,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:31,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +18: [2023-05-13 09:38:31,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:31,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +28: [2023-05-13 09:38:31,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +30: [2023-05-13 09:38:31,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:31,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:31,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:31,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +18: [2023-05-13 09:38:31,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +27: [2023-05-13 09:38:31,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:31,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +18: [2023-05-13 09:38:31,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +18: [2023-05-13 09:38:31,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +27: [2023-05-13 09:38:31,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +22: [2023-05-13 09:38:31,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:31,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:31,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:31,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:31,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:31,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:31,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:31,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:31,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:31,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +22: [2023-05-13 09:38:31,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +22: [2023-05-13 09:38:31,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:31,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:31,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:31,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:31,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:31,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:31,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:31,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 5: [2023-05-13 09:38:31,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:31,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:31,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:31,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:31,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 8: [2023-05-13 09:38:31,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:31,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:31,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:31,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:31,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:31,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +29: [2023-05-13 09:38:31,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:31,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:31,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:31,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:31,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:31,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:31,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:31,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:31,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:31,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:31,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:31,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:31,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:31,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:31,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:31,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:31,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:31,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:31,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:31,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:31,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:31,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:31,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:31,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:31,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:31,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:31,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:31,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:31,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:31,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:31,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:31,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:31,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:31,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:31,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:31,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:31,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:31,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:31,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +20: [2023-05-13 09:38:31,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:31,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:31,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +20: [2023-05-13 09:38:31,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:31,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:31,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:31,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:31,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:31,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:31,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:31,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +19: [2023-05-13 09:38:31,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +19: [2023-05-13 09:38:31,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 7: [2023-05-13 09:38:31,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:31,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +19: [2023-05-13 09:38:31,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:31,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:31,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:31,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:31,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:31,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:31,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 5: [2023-05-13 09:38:31,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:31,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:31,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:31,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:31,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:31,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:31,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:31,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:31,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +26: [2023-05-13 09:38:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +26: [2023-05-13 09:38:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +26: [2023-05-13 09:38:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:31,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +24: [2023-05-13 09:38:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +24: [2023-05-13 09:38:31,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:31,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:31,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:31,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:31,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:31,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:31,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:31,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:31,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:31,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:31,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:31,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:31,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:31,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:31,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:31,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:31,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:31,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:31,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:31,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:31,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:31,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:31,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:31,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:31,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:31,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:31,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:31,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:31,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:31,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:31,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:31,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:31,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:31,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:31,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:31,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +15: [2023-05-13 09:38:31,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:31,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:31,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:31,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:31,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:31,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:31,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:31,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:31,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:31,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:31,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:31,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:31,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:31,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:31,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. + 0: [2023-05-13 09:38:31,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +15: [2023-05-13 09:38:31,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:31,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_05-model_01-model_states.pt. +15: [2023-05-13 09:38:31,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:31,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:31,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:31,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:31,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:31,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +27: [2023-05-13 09:38:31,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:31,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:31,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:31,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 5: [2023-05-13 09:38:31,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +27: [2023-05-13 09:38:31,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:31,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:31,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:31,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:31,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:31,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:31,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:31,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:31,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:31,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:31,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:31,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:31,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:31,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:31,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:31,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:31,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:31,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:31,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:31,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:31,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:31,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:31,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:31,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:31,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:31,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:31,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:31,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:31,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:31,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:31,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:31,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:31,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:31,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:31,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:31,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +19: [2023-05-13 09:38:31,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +16: [2023-05-13 09:38:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:31,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:31,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:31,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:31,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +22: [2023-05-13 09:38:31,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:31,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:31,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:31,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:31,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:31,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:31,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:31,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:31,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:31,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:31,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:31,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:31,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:31,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:31,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +26: [2023-05-13 09:38:31,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:31,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:31,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:31,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +22: [2023-05-13 09:38:31,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:31,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:31,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:31,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:31,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:31,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:31,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:31,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:31,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:31,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:31,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:31,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:31,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:31,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:31,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:31,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:31,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 8: [2023-05-13 09:38:31,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:31,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:31,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:31,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:31,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:31,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:31,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:31,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:31,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:31,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:31,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:31,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:31,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:31,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:31,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:31,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:31,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:31,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:31,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:31,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +26: [2023-05-13 09:38:31,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:31,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:31,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:31,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:31,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:31,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:31,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 2: [2023-05-13 09:38:31,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:31,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:31,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:31,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:31,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:31,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:31,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:31,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:31,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:31,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:31,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:31,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:31,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:31,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:31,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:31,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:31,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:31,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:31,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:31,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:31,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:31,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:31,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:31,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +26: [2023-05-13 09:38:31,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 6: [2023-05-13 09:38:31,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:31,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:31,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:31,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +13: [2023-05-13 09:38:31,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +13: [2023-05-13 09:38:31,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:31,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +13: [2023-05-13 09:38:31,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +13: [2023-05-13 09:38:31,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:31,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:31,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:31,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:31,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:31,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:31,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:31,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:31,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:31,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:31,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:31,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:31,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:31,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:31,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:31,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:31,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:31,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:31,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:31,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +17: [2023-05-13 09:38:31,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:31,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +30: [2023-05-13 09:38:31,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 3: [2023-05-13 09:38:31,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +30: [2023-05-13 09:38:31,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:31,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:31,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:31,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:31,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 4: [2023-05-13 09:38:31,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:31,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +12: [2023-05-13 09:38:32,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:32,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +12: [2023-05-13 09:38:32,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +20: [2023-05-13 09:38:32,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +20: [2023-05-13 09:38:32,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +20: [2023-05-13 09:38:32,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:32,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +29: [2023-05-13 09:38:32,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +29: [2023-05-13 09:38:32,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +29: [2023-05-13 09:38:32,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:32,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:32,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:32,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:32,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:32,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:32,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:32,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:32,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:32,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:32,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:32,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:32,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:32,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:32,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:32,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +20: [2023-05-13 09:38:32,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:32,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:32,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:32,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +28: [2023-05-13 09:38:32,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +28: [2023-05-13 09:38:32,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:32,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:32,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:32,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:32,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:32,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:32,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:32,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +25: [2023-05-13 09:38:32,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +17: [2023-05-13 09:38:32,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +17: [2023-05-13 09:38:32,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:32,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:32,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:32,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:32,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:32,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:32,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:32,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:32,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:32,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:32,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +12: [2023-05-13 09:38:32,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:32,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +20: [2023-05-13 09:38:32,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:32,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:32,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +21: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +21: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +21: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:32,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +20: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +18: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:32,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +20: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 8: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 8: [2023-05-13 09:38:32,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +20: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:32,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +20: [2023-05-13 09:38:32,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:32,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +29: [2023-05-13 09:38:32,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +31: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +31: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +29: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:32,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:32,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:32,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:32,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:32,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:32,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:32,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +18: [2023-05-13 09:38:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +29: [2023-05-13 09:38:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +25: [2023-05-13 09:38:32,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:32,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:32,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:32,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:32,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:32,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:32,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +14: [2023-05-13 09:38:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +14: [2023-05-13 09:38:32,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:32,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:32,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +16: [2023-05-13 09:38:32,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +16: [2023-05-13 09:38:32,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:32,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +22: [2023-05-13 09:38:32,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +28: [2023-05-13 09:38:32,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:32,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:32,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:32,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:32,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +23: [2023-05-13 09:38:32,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:32,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +15: [2023-05-13 09:38:32,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:32,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:32,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:32,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:32,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +25: [2023-05-13 09:38:32,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:32,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:32,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:32,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:32,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:32,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:32,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:32,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:32,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:32,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:32,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:32,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:32,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:32,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +27: [2023-05-13 09:38:32,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:32,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +18: [2023-05-13 09:38:32,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:32,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +21: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:32,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:32,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:32,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:32,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:32,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:32,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:32,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:32,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:32,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:32,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:32,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:32,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:32,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:32,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:32,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:32,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:32,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +19: [2023-05-13 09:38:32,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:32,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:32,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +19: [2023-05-13 09:38:32,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +19: [2023-05-13 09:38:32,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +14: [2023-05-13 09:38:32,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:32,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +11: [2023-05-13 09:38:32,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +11: [2023-05-13 09:38:32,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +11: [2023-05-13 09:38:32,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:32,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:32,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:32,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:32,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:32,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +16: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +14: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +14: [2023-05-13 09:38:32,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +15: [2023-05-13 09:38:32,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:32,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:32,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +15: [2023-05-13 09:38:32,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +15: [2023-05-13 09:38:32,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:32,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:32,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:32,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:32,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:32,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:32,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:32,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:32,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:32,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:32,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:32,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:32,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:32,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +25: [2023-05-13 09:38:32,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +25: [2023-05-13 09:38:32,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +25: [2023-05-13 09:38:32,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:32,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:32,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:32,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:32,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:32,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:32,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:32,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:32,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:32,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:32,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:32,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:32,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:32,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:32,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:32,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:32,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +19: [2023-05-13 09:38:32,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:32,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +19: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:32,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +24: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +12: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_06-model_01-model_states.pt. +24: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:32,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:32,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:32,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:32,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:32,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:32,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:32,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:32,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:32,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:32,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +24: [2023-05-13 09:38:32,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +24: [2023-05-13 09:38:32,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +24: [2023-05-13 09:38:32,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:32,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:32,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:32,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:32,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:32,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:32,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:32,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:32,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:32,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:32,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:32,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:32,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:32,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:32,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:32,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:32,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:32,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:32,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:32,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:32,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:32,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:32,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +19: [2023-05-13 09:38:32,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:32,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:32,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:32,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +26: [2023-05-13 09:38:32,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +26: [2023-05-13 09:38:32,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +19: [2023-05-13 09:38:32,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:32,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +26: [2023-05-13 09:38:32,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:32,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:32,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:32,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:32,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:32,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:32,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:32,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:32,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:32,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:32,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 2: [2023-05-13 09:38:32,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 2: [2023-05-13 09:38:32,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 2: [2023-05-13 09:38:32,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 2: [2023-05-13 09:38:32,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 2: [2023-05-13 09:38:32,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:32,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:32,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:32,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:32,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:32,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:32,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:32,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:32,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:32,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:32,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:32,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:32,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:32,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:32,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:32,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:32,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:32,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:32,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:32,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +26: [2023-05-13 09:38:32,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:32,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:32,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:32,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +27: [2023-05-13 09:38:32,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:32,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:32,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:32,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:32,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:32,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +21: [2023-05-13 09:38:32,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:32,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:32,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:32,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:32,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:32,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:32,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:32,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +12: [2023-05-13 09:38:32,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:32,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:32,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:32,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:32,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:32,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:32,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:32,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:32,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +21: [2023-05-13 09:38:32,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:32,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:32,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +20: [2023-05-13 09:38:32,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +20: [2023-05-13 09:38:32,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +20: [2023-05-13 09:38:32,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:32,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:32,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:32,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:32,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:32,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:32,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:32,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:32,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:32,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:32,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:32,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:32,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:32,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:32,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:32,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:32,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:32,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:32,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:32,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:32,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:32,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:32,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:32,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:32,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:32,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:32,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:32,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:32,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:32,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:32,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:32,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:32,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:32,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:32,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:32,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:32,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +12: [2023-05-13 09:38:32,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:32,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +11: [2023-05-13 09:38:32,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +11: [2023-05-13 09:38:32,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +11: [2023-05-13 09:38:32,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:32,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:32,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:32,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +27: [2023-05-13 09:38:32,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +18: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:32,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:32,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:32,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:32,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +29: [2023-05-13 09:38:32,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:32,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +16: [2023-05-13 09:38:32,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:32,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:32,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:32,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:32,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +16: [2023-05-13 09:38:32,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:32,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:32,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:32,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:32,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:32,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:32,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:32,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:32,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:32,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:32,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:32,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:32,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:32,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +17: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +17: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +16: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:32,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +20: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:32,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:32,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:32,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:32,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:32,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:32,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:32,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:32,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:32,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:32,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:32,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:32,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:32,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:32,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:32,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:32,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:32,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:32,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:32,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:32,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:32,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:32,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:32,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:32,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:32,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:32,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:32,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +18: [2023-05-13 09:38:32,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +10: [2023-05-13 09:38:32,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +10: [2023-05-13 09:38:32,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +10: [2023-05-13 09:38:32,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:32,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +18: [2023-05-13 09:38:32,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +19: [2023-05-13 09:38:32,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +19: [2023-05-13 09:38:32,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +11: [2023-05-13 09:38:32,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:32,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:32,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +20: [2023-05-13 09:38:32,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:32,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:32,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +20: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:32,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:32,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:32,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:32,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:32,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:32,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:32,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:32,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:32,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:32,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +11: [2023-05-13 09:38:32,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:32,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +13: [2023-05-13 09:38:32,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:32,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:32,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:32,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +10: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:32,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_07-model_01-model_states.pt. +26: [2023-05-13 09:38:32,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:32,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:32,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:32,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:32,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:32,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:32,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:32,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:32,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:32,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:32,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:32,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:32,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +13: [2023-05-13 09:38:32,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:32,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:32,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +13: [2023-05-13 09:38:32,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:32,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:32,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:32,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:32,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:32,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:32,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 8: [2023-05-13 09:38:32,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 8: [2023-05-13 09:38:32,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:32,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:32,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:32,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:32,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:32,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:32,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:32,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:32,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +19: [2023-05-13 09:38:32,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +19: [2023-05-13 09:38:32,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +19: [2023-05-13 09:38:32,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:32,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:32,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:32,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:32,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:32,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +23: [2023-05-13 09:38:32,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:32,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +24: [2023-05-13 09:38:32,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:32,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:32,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:32,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +19: [2023-05-13 09:38:32,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +24: [2023-05-13 09:38:32,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:32,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +10: [2023-05-13 09:38:32,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:32,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +10: [2023-05-13 09:38:32,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:32,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +10: [2023-05-13 09:38:32,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:32,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:32,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:32,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:32,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +21: [2023-05-13 09:38:32,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:32,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:32,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:32,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:32,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +29: [2023-05-13 09:38:32,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +29: [2023-05-13 09:38:32,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +29: [2023-05-13 09:38:32,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +29: [2023-05-13 09:38:32,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:32,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:32,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:32,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:32,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:32,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:32,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:32,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:32,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:32,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:32,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:32,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:32,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:32,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:32,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:32,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +28: [2023-05-13 09:38:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +28: [2023-05-13 09:38:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +28: [2023-05-13 09:38:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +28: [2023-05-13 09:38:32,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:32,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:32,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:32,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +24: [2023-05-13 09:38:32,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:32,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:32,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:32,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +23: [2023-05-13 09:38:32,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +23: [2023-05-13 09:38:32,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:32,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:32,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +24: [2023-05-13 09:38:32,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:32,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:32,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:32,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:32,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:32,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:32,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:32,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:32,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:32,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +28: [2023-05-13 09:38:32,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:32,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:32,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:32,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +21: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +24: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +24: [2023-05-13 09:38:32,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:32,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +21: [2023-05-13 09:38:32,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:32,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +25: [2023-05-13 09:38:32,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:32,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:32,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:32,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:32,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +25: [2023-05-13 09:38:32,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:32,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:32,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:32,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:32,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:32,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:32,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:32,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:32,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:32,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:32,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:32,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +28: [2023-05-13 09:38:32,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:32,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:32,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:32,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:32,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +28: [2023-05-13 09:38:32,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:32,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:32,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:32,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:32,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +18: [2023-05-13 09:38:32,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:32,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:32,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:32,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:32,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:32,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:32,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:32,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 2: [2023-05-13 09:38:32,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +18: [2023-05-13 09:38:32,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:32,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:32,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:32,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 2: [2023-05-13 09:38:32,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 2: [2023-05-13 09:38:32,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:32,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:32,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:32,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +24: [2023-05-13 09:38:32,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:32,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +24: [2023-05-13 09:38:32,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:32,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:32,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:32,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:32,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:32,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:32,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:32,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +24: [2023-05-13 09:38:32,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:32,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +20: [2023-05-13 09:38:32,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +20: [2023-05-13 09:38:32,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +20: [2023-05-13 09:38:32,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +24: [2023-05-13 09:38:32,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +17: [2023-05-13 09:38:32,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:32,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:32,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:32,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:32,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:32,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:32,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +13: [2023-05-13 09:38:32,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:32,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:32,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:32,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +26: [2023-05-13 09:38:32,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:32,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:32,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:32,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:32,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:32,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:32,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:32,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +11: [2023-05-13 09:38:32,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +11: [2023-05-13 09:38:32,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +11: [2023-05-13 09:38:32,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +11: [2023-05-13 09:38:32,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +12: [2023-05-13 09:38:32,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:32,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:32,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:32,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:32,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +12: [2023-05-13 09:38:32,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +30: [2023-05-13 09:38:32,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +30: [2023-05-13 09:38:32,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +12: [2023-05-13 09:38:32,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:32,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:32,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:32,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:32,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:32,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +13: [2023-05-13 09:38:32,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:32,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:32,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:32,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +14: [2023-05-13 09:38:32,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:32,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:32,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:32,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:32,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:32,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:32,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:32,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:32,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:32,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:32,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:32,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:32,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:32,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:32,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:32,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:32,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:32,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:32,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:32,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:32,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:32,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +14: [2023-05-13 09:38:32,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +14: [2023-05-13 09:38:32,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:32,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +14: [2023-05-13 09:38:32,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +20: [2023-05-13 09:38:32,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:32,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:32,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +20: [2023-05-13 09:38:32,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +27: [2023-05-13 09:38:32,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:32,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:32,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:32,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:32,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:32,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:32,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:32,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:32,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:32,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:32,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:32,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:32,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:32,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:32,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:32,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:32,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:32,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:32,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:32,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:32,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:32,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:32,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:32,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:32,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:32,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:32,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:32,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:32,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:32,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:32,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:32,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:32,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:32,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:32,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:32,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:32,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:32,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:32,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:32,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:32,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +12: [2023-05-13 09:38:32,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:32,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:32,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:32,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:32,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:32,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:32,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:32,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:32,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:32,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:32,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:32,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:32,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:32,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:32,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:32,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:32,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:32,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +17: [2023-05-13 09:38:32,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:32,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:32,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:32,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:32,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:32,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:32,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:32,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:32,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:32,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:32,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:32,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +11: [2023-05-13 09:38:32,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:32,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +12: [2023-05-13 09:38:32,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +14: [2023-05-13 09:38:32,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:32,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:32,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +17: [2023-05-13 09:38:32,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:32,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:32,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:32,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:32,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:32,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:32,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:32,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +12: [2023-05-13 09:38:32,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:32,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +11: [2023-05-13 09:38:32,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:32,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:32,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +27: [2023-05-13 09:38:32,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:32,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +31: [2023-05-13 09:38:32,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:32,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:32,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:32,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:32,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +31: [2023-05-13 09:38:32,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:32,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:32,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:32,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:32,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:32,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:32,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:32,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:32,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:32,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +30: [2023-05-13 09:38:32,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +12: [2023-05-13 09:38:32,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:32,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:32,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:32,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:32,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:32,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:32,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:32,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:32,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:32,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:32,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:32,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:32,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:32,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:32,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:32,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:32,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:32,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:32,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:32,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:32,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:32,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:32,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:32,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:32,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:32,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:32,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:32,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:32,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:32,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:32,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:32,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +14: [2023-05-13 09:38:32,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:32,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:32,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:32,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:32,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:32,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:32,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:32,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +23: [2023-05-13 09:38:32,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:32,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:32,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:32,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:32,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +14: [2023-05-13 09:38:32,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +14: [2023-05-13 09:38:32,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:32,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:32,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:32,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:32,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:32,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:32,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:32,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:32,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:32,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:32,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:32,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:32,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:32,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:32,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:32,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:32,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. + 4: [2023-05-13 09:38:32,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +28: [2023-05-13 09:38:32,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:32,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:32,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:32,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:32,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:32,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:32,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:32,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:32,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:32,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:32,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:32,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:32,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:32,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:32,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:32,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:32,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:32,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:32,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:32,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:32,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:32,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:32,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:32,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:32,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +29: [2023-05-13 09:38:32,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +29: [2023-05-13 09:38:32,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:32,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:32,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:32,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:32,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:32,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:32,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:32,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:32,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:32,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:32,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:32,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:32,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:32,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:32,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +12: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:32,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:32,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +23: [2023-05-13 09:38:32,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:32,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:32,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:32,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:32,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:32,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 9: [2023-05-13 09:38:32,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:32,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:32,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:32,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:32,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:32,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +26: [2023-05-13 09:38:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:32,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:32,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:32,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:32,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:32,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:32,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:32,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:32,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:32,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:32,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:32,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:32,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:32,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:32,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +27: [2023-05-13 09:38:32,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:32,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:32,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:32,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:32,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:32,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +27: [2023-05-13 09:38:32,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:32,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:32,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:32,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:32,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:32,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:32,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:32,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:32,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:32,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:32,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:32,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:32,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:32,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:32,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:32,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:32,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:32,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +24: [2023-05-13 09:38:32,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:32,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:32,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:32,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:32,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:32,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:32,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:32,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:32,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:32,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:32,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:32,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:32,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:32,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:32,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +29: [2023-05-13 09:38:32,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:32,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:32,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:32,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:32,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:32,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:32,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:32,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:32,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:32,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:32,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:32,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:32,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:32,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +31: [2023-05-13 09:38:32,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:32,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:32,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:32,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:32,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:32,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:32,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:32,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:32,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:32,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:32,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:32,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:32,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:32,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:32,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:32,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:32,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:32,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +11: [2023-05-13 09:38:32,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:32,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:32,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:32,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:32,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:32,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:32,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:32,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:32,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:32,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:32,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:32,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:32,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:32,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:32,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:32,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:32,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:32,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:32,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +29: [2023-05-13 09:38:32,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +29: [2023-05-13 09:38:32,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +29: [2023-05-13 09:38:32,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt... +24: [2023-05-13 09:38:32,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:32,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:32,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:32,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:32,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:32,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:32,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:32,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:32,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +23: [2023-05-13 09:38:32,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +23: [2023-05-13 09:38:32,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +23: [2023-05-13 09:38:32,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +23: [2023-05-13 09:38:32,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +24: [2023-05-13 09:38:32,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:32,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:32,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:32,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:32,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:32,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:32,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:32,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:32,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:32,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:32,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:32,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 3: [2023-05-13 09:38:32,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:32,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:32,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:32,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:32,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:32,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:32,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:32,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:32,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:32,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:32,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:32,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:32,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:32,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:32,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:32,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:32,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:32,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:32,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:32,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:32,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:32,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 7: [2023-05-13 09:38:32,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:32,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:32,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:32,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:32,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:32,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:32,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:32,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:32,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:32,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:32,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +23: [2023-05-13 09:38:32,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +23: [2023-05-13 09:38:32,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +23: [2023-05-13 09:38:32,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:32,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:32,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:32,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +18: [2023-05-13 09:38:32,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +18: [2023-05-13 09:38:32,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:32,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:32,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +18: [2023-05-13 09:38:32,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:32,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:32,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:32,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:32,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:32,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:32,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:32,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:32,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:32,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:32,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:32,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:32,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:32,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:32,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:32,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:32,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:32,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:32,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +24: [2023-05-13 09:38:32,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +24: [2023-05-13 09:38:32,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +24: [2023-05-13 09:38:32,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:32,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:32,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:32,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:32,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:32,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:32,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:32,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:32,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:32,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:32,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +24: [2023-05-13 09:38:32,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:32,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:32,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:32,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +26: [2023-05-13 09:38:33,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +26: [2023-05-13 09:38:33,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:33,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:33,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:33,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:33,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:33,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +16: [2023-05-13 09:38:33,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:33,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +16: [2023-05-13 09:38:33,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +24: [2023-05-13 09:38:33,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:33,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:33,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:33,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:33,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +24: [2023-05-13 09:38:33,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +24: [2023-05-13 09:38:33,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:33,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:33,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:33,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:33,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:33,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:33,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:33,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:33,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:33,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:33,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +19: [2023-05-13 09:38:33,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:33,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:33,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:33,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +22: [2023-05-13 09:38:33,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:33,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:33,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +31: [2023-05-13 09:38:33,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:33,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:33,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:33,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:33,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:33,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:33,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:33,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:33,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:33,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +15: [2023-05-13 09:38:33,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +15: [2023-05-13 09:38:33,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:33,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:33,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:33,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:33,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:33,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:33,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:33,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:33,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:33,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:33,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:33,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:33,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:33,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:33,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:33,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:33,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:33,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:33,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:33,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:33,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:33,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:33,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:33,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:33,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +15: [2023-05-13 09:38:33,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +25: [2023-05-13 09:38:33,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:33,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +25: [2023-05-13 09:38:33,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:33,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:33,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:33,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:33,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +17: [2023-05-13 09:38:33,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +22: [2023-05-13 09:38:33,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:33,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:33,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:33,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +25: [2023-05-13 09:38:33,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +22: [2023-05-13 09:38:33,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +10: [2023-05-13 09:38:33,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +22: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:33,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:33,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +30: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +10: [2023-05-13 09:38:33,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +10: [2023-05-13 09:38:33,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +20: [2023-05-13 09:38:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:33,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:33,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:33,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +20: [2023-05-13 09:38:33,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:33,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:33,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:33,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +20: [2023-05-13 09:38:33,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +20: [2023-05-13 09:38:33,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +20: [2023-05-13 09:38:33,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:33,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:33,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:33,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +15: [2023-05-13 09:38:33,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +12: [2023-05-13 09:38:33,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:33,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:33,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:33,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:33,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +12: [2023-05-13 09:38:33,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:33,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:33,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:33,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:33,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:33,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:33,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:33,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +29: [2023-05-13 09:38:33,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +29: [2023-05-13 09:38:33,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:33,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +26: [2023-05-13 09:38:33,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:33,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +19: [2023-05-13 09:38:33,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:33,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:33,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +19: [2023-05-13 09:38:33,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +29: [2023-05-13 09:38:33,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:33,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:33,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:33,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +17: [2023-05-13 09:38:33,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +17: [2023-05-13 09:38:33,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:33,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:33,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:33,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:33,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:33,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:33,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:33,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +12: [2023-05-13 09:38:33,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:33,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +18: [2023-05-13 09:38:33,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:33,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:33,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:33,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:33,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:33,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:33,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:33,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:33,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:33,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:33,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:33,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:33,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:33,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +10: [2023-05-13 09:38:33,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +22: [2023-05-13 09:38:33,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:33,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +14: [2023-05-13 09:38:33,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +31: [2023-05-13 09:38:33,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:33,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +14: [2023-05-13 09:38:33,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:33,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:33,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:33,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:33,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:33,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:33,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:33,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:33,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:33,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:33,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:33,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:33,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:33,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:33,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:33,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:33,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:33,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:33,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:33,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:33,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:33,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:33,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +15: [2023-05-13 09:38:33,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:33,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +19: [2023-05-13 09:38:33,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:33,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:33,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:33,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:33,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:33,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:33,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:33,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:33,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:33,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:33,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:33,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:33,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:33,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:33,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +22: [2023-05-13 09:38:33,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:33,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:33,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:33,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:33,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:33,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:33,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:33,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:33,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:33,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:33,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:33,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:33,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:33,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:33,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:33,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +18: [2023-05-13 09:38:33,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:33,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +18: [2023-05-13 09:38:33,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:33,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +20: [2023-05-13 09:38:33,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:33,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:33,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:33,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:33,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:33,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:33,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:33,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:33,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +23: [2023-05-13 09:38:33,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:33,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +11: [2023-05-13 09:38:33,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:33,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:33,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:33,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:33,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:33,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:33,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:33,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:33,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +11: [2023-05-13 09:38:33,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +11: [2023-05-13 09:38:33,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +23: [2023-05-13 09:38:33,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:33,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:33,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:33,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:33,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:33,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +16: [2023-05-13 09:38:33,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +16: [2023-05-13 09:38:33,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +16: [2023-05-13 09:38:33,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +23: [2023-05-13 09:38:33,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:33,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:33,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:33,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +11: [2023-05-13 09:38:33,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:33,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:33,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:33,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:33,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +24: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +12: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +24: [2023-05-13 09:38:33,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:33,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +18: [2023-05-13 09:38:33,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:33,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +23: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +23: [2023-05-13 09:38:33,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +23: [2023-05-13 09:38:33,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:33,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:33,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:33,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:33,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:33,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:33,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:33,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:33,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:33,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +16: [2023-05-13 09:38:33,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:33,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:33,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:33,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +16: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:33,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +21: [2023-05-13 09:38:33,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +21: [2023-05-13 09:38:33,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +21: [2023-05-13 09:38:33,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:33,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:33,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:33,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:33,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:33,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:33,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:33,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:33,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:33,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +30: [2023-05-13 09:38:33,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:33,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:33,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:33,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:33,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:33,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:33,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:33,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:33,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:33,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +21: [2023-05-13 09:38:33,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:33,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +21: [2023-05-13 09:38:33,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:33,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:33,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:33,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:33,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:33,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:33,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:33,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +17: [2023-05-13 09:38:33,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:33,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +14: [2023-05-13 09:38:33,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +28: [2023-05-13 09:38:33,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +28: [2023-05-13 09:38:33,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:33,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:33,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:33,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +28: [2023-05-13 09:38:33,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:33,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +21: [2023-05-13 09:38:33,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:33,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +17: [2023-05-13 09:38:33,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:33,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:33,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:33,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:33,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +26: [2023-05-13 09:38:33,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:33,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:33,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:33,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:33,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +28: [2023-05-13 09:38:33,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:33,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:33,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:33,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:33,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +25: [2023-05-13 09:38:33,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:33,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:33,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:33,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:33,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +25: [2023-05-13 09:38:33,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:33,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:33,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +28: [2023-05-13 09:38:33,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:33,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:33,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:33,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +25: [2023-05-13 09:38:33,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +28: [2023-05-13 09:38:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +10: [2023-05-13 09:38:33,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:33,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:33,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +10: [2023-05-13 09:38:33,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:33,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:33,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:33,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:33,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:33,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +28: [2023-05-13 09:38:33,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:33,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:33,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:33,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:33,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:33,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +30: [2023-05-13 09:38:33,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:33,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:33,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:33,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:33,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +26: [2023-05-13 09:38:33,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +26: [2023-05-13 09:38:33,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:33,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:33,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:33,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:33,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:33,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +21: [2023-05-13 09:38:33,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:33,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +26: [2023-05-13 09:38:33,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:33,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:33,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:33,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:33,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:33,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:33,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:33,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:33,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:33,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:33,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:33,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:33,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:33,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:33,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:33,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:33,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:33,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:33,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:33,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:33,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:33,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:33,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:33,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:33,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:33,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:33,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:33,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:33,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:33,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:33,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:33,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:33,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +20: [2023-05-13 09:38:33,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:33,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:33,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:33,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +20: [2023-05-13 09:38:33,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:33,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +30: [2023-05-13 09:38:33,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +20: [2023-05-13 09:38:33,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +21: [2023-05-13 09:38:33,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:33,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:33,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:33,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:33,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +10: [2023-05-13 09:38:33,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:33,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:33,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:33,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +12: [2023-05-13 09:38:33,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:33,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:33,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:33,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +12: [2023-05-13 09:38:33,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:33,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:33,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:33,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:33,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:33,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:33,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:33,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:33,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:33,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:33,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:33,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:33,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:33,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:33,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:33,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:33,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:33,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:33,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:33,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:33,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:33,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:33,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:33,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:33,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:33,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:33,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:33,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:33,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:33,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:33,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:33,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:33,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:33,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:33,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:33,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:33,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:33,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:33,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:33,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +27: [2023-05-13 09:38:33,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +26: [2023-05-13 09:38:33,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:33,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:33,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:33,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +11: [2023-05-13 09:38:33,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:33,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:33,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:33,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:33,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:33,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:33,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:33,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:33,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:33,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:33,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +13: [2023-05-13 09:38:33,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +13: [2023-05-13 09:38:33,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:33,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +13: [2023-05-13 09:38:33,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +13: [2023-05-13 09:38:33,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +27: [2023-05-13 09:38:33,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:33,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +20: [2023-05-13 09:38:33,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:33,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +20: [2023-05-13 09:38:33,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +20: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:33,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:33,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:33,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +27: [2023-05-13 09:38:33,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:33,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +13: [2023-05-13 09:38:33,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:33,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:33,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:33,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +12: [2023-05-13 09:38:33,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:33,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:33,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:33,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:33,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:33,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:33,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:33,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +10: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +11: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +11: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +11: [2023-05-13 09:38:33,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:33,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:33,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +12: [2023-05-13 09:38:33,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +12: [2023-05-13 09:38:33,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:33,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:33,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +12: [2023-05-13 09:38:33,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:33,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +15: [2023-05-13 09:38:33,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +15: [2023-05-13 09:38:33,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. +15: [2023-05-13 09:38:33,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:33,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:33,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:33,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:33,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:33,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:33,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:33,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:33,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:33,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:33,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:33,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:33,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:33,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:33,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:33,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:33,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +13: [2023-05-13 09:38:33,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:33,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:33,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:33,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:33,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +19: [2023-05-13 09:38:33,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:33,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +13: [2023-05-13 09:38:33,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:33,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +13: [2023-05-13 09:38:33,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +17: [2023-05-13 09:38:33,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +17: [2023-05-13 09:38:33,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:33,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +19: [2023-05-13 09:38:33,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +17: [2023-05-13 09:38:33,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +19: [2023-05-13 09:38:33,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:33,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:33,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:33,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:33,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:33,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:33,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:33,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:33,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:33,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +14: [2023-05-13 09:38:33,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +14: [2023-05-13 09:38:33,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +14: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +14: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +14: [2023-05-13 09:38:33,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:33,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:33,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:33,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +10: [2023-05-13 09:38:33,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:33,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +10: [2023-05-13 09:38:33,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +27: [2023-05-13 09:38:33,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +27: [2023-05-13 09:38:33,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:33,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:33,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:33,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:33,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:33,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:33,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:33,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:33,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:33,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:33,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:33,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:33,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +27: [2023-05-13 09:38:33,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:33,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:33,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:33,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:33,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +18: [2023-05-13 09:38:33,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +15: [2023-05-13 09:38:33,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +13: [2023-05-13 09:38:33,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:33,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:33,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:33,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +18: [2023-05-13 09:38:33,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:33,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:33,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:33,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:33,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:33,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:33,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:33,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:33,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:33,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:33,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:33,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:33,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:33,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:33,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:33,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:33,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:33,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:33,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:33,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:33,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:33,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:33,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:33,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:33,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:33,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:33,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:33,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:33,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:33,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:33,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:33,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:33,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:33,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:33,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:33,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:33,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:33,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:33,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:33,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:33,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:33,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:33,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:33,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:33,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:33,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:33,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:33,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:33,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:33,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:33,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +12: [2023-05-13 09:38:33,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:33,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:33,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:33,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:33,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:33,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:33,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:33,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:33,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:33,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:33,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:33,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:33,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:33,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +18: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +16: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +16: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +16: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:33,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:33,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:33,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:33,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:33,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:33,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:33,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:33,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:33,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 9: [2023-05-13 09:38:33,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:33,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:33,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:33,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:33,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:33,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:33,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:33,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:33,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:33,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:33,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:33,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:33,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:33,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:33,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:33,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:33,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:33,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:33,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:33,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 6: [2023-05-13 09:38:33,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:33,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +23: [2023-05-13 09:38:33,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +23: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +23: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:33,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:33,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:33,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:33,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:33,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:33,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +16: [2023-05-13 09:38:33,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:33,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:33,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:33,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:33,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:33,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:33,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:33,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:33,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:33,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:33,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:33,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:33,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:33,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:33,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:33,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:33,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:33,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:33,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:33,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 1: [2023-05-13 09:38:33,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:33,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:33,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:33,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:33,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:33,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:33,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:33,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:33,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:33,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +26: [2023-05-13 09:38:33,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +26: [2023-05-13 09:38:33,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +26: [2023-05-13 09:38:33,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:33,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:33,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:33,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:33,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:33,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:33,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:33,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:33,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:33,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:33,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:33,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:33,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:33,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:33,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:33,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +28: [2023-05-13 09:38:33,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:33,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +24: [2023-05-13 09:38:33,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +24: [2023-05-13 09:38:33,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:33,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:33,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:33,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +24: [2023-05-13 09:38:33,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +26: [2023-05-13 09:38:33,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:33,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:33,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:33,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:33,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:33,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:33,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:33,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:33,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:33,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:33,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:33,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:33,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:33,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:33,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:33,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:33,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:33,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +16: [2023-05-13 09:38:33,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:33,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:33,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:33,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:33,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:33,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:33,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +24: [2023-05-13 09:38:33,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:33,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:33,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:33,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:33,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:33,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:33,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:33,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:33,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:33,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:33,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:33,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:33,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:33,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:33,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +21: [2023-05-13 09:38:33,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:33,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:33,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +21: [2023-05-13 09:38:33,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:33,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:33,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:33,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:33,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:33,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:33,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +26: [2023-05-13 09:38:33,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:33,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:33,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:33,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:33,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:33,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +24: [2023-05-13 09:38:33,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:33,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:33,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +24: [2023-05-13 09:38:33,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:33,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:33,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:33,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:33,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +24: [2023-05-13 09:38:33,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +24: [2023-05-13 09:38:33,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:33,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:33,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:33,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:33,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:33,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:33,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:33,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:33,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:33,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:33,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 3: [2023-05-13 09:38:33,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:33,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:33,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:33,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:33,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:33,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:33,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:33,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:33,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:33,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:33,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:33,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:33,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:33,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:33,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:33,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +20: [2023-05-13 09:38:33,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +20: [2023-05-13 09:38:33,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +25: [2023-05-13 09:38:33,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +25: [2023-05-13 09:38:33,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +25: [2023-05-13 09:38:33,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +20: [2023-05-13 09:38:33,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +25: [2023-05-13 09:38:33,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +27: [2023-05-13 09:38:33,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:33,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:33,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +27: [2023-05-13 09:38:33,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:33,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:33,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:33,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:33,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:33,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:33,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:33,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:33,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:33,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:33,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:33,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:33,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:33,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:33,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:33,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:33,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:33,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:33,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:33,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:33,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:33,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:33,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:33,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:33,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:33,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:33,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:33,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:33,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:33,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:33,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:33,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:33,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:33,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:33,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:33,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:33,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:33,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:33,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:33,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:33,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:33,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:33,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:33,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:33,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:33,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:33,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:33,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:33,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:33,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:33,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:33,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:33,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:33,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:33,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:33,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:33,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:33,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:33,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:33,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:33,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:33,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:33,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:33,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:33,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +15: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:33,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:33,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:33,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:33,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:33,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:33,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:33,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:33,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:33,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:33,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:33,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:33,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:33,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:33,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:33,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:33,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +13: [2023-05-13 09:38:33,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:33,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:33,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:33,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:33,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:33,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:33,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:33,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:33,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +20: [2023-05-13 09:38:33,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:33,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +20: [2023-05-13 09:38:33,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:33,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +27: [2023-05-13 09:38:33,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +31: [2023-05-13 09:38:33,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:33,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +20: [2023-05-13 09:38:33,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +31: [2023-05-13 09:38:33,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:33,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +28: [2023-05-13 09:38:33,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:33,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:33,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:33,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:33,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:33,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:33,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:33,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:33,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:33,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:33,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:33,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:33,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:33,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:33,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:33,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:33,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:33,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:33,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:33,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:33,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:33,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:33,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:33,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:33,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:33,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:33,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:33,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:33,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:33,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:33,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:33,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:33,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:33,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:33,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 0: [2023-05-13 09:38:33,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:33,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:33,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:33,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:33,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:33,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:33,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:33,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:33,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:33,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:33,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:33,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:33,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:33,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:33,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:33,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:33,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:33,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:33,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:33,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:33,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:33,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 1: [2023-05-13 09:38:33,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +15: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +11: [2023-05-13 09:38:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +11: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +11: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:33,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +14: [2023-05-13 09:38:33,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:33,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:33,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:33,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:33,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:33,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +15: [2023-05-13 09:38:33,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:33,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:33,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:33,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:33,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:33,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:33,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:33,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:33,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:33,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:33,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:33,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:33,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:33,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:33,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:33,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:33,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:33,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:33,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +27: [2023-05-13 09:38:33,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:33,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 2: [2023-05-13 09:38:33,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 2: [2023-05-13 09:38:33,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +24: [2023-05-13 09:38:33,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:33,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:33,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:33,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:33,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:33,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +27: [2023-05-13 09:38:33,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +24: [2023-05-13 09:38:33,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:33,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:34,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +24: [2023-05-13 09:38:34,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:34,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:34,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:34,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:34,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:34,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:34,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:34,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +19: [2023-05-13 09:38:34,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:34,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:34,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:34,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:34,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:34,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:34,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:34,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:34,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:34,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +14: [2023-05-13 09:38:34,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +14: [2023-05-13 09:38:34,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +14: [2023-05-13 09:38:34,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:34,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:34,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:34,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +14: [2023-05-13 09:38:34,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:34,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:34,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +22: [2023-05-13 09:38:34,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:34,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:34,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:34,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:34,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:34,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:34,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:34,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:34,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:34,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:34,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +11: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:34,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:34,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:34,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:34,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:34,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:34,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +24: [2023-05-13 09:38:34,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:34,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:34,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:34,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:34,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:34,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:34,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:34,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:34,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:34,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +27: [2023-05-13 09:38:34,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +27: [2023-05-13 09:38:34,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:34,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +29: [2023-05-13 09:38:34,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +29: [2023-05-13 09:38:34,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +29: [2023-05-13 09:38:34,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:34,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:34,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:34,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:34,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:34,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:34,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:34,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:34,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:34,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:34,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +27: [2023-05-13 09:38:34,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:34,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:34,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:34,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:34,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:34,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +15: [2023-05-13 09:38:34,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:34,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +24: [2023-05-13 09:38:34,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:34,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:34,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +15: [2023-05-13 09:38:34,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:34,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:34,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +12: [2023-05-13 09:38:34,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:34,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:34,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:34,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +17: [2023-05-13 09:38:34,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:34,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:34,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:34,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +18: [2023-05-13 09:38:34,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:34,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:34,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:34,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:34,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:34,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +13: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:34,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:34,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:34,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:34,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:34,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +14: [2023-05-13 09:38:34,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:34,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:34,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +15: [2023-05-13 09:38:34,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:34,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:34,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:34,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:34,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:34,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:34,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +17: [2023-05-13 09:38:34,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +15: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:34,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:34,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:34,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:34,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:34,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 6: [2023-05-13 09:38:34,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 6: [2023-05-13 09:38:34,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +23: [2023-05-13 09:38:34,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:34,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +31: [2023-05-13 09:38:34,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:34,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +31: [2023-05-13 09:38:34,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +18: [2023-05-13 09:38:34,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +18: [2023-05-13 09:38:34,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +18: [2023-05-13 09:38:34,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:34,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:34,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:34,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +15: [2023-05-13 09:38:34,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +15: [2023-05-13 09:38:34,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +17: [2023-05-13 09:38:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:34,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:34,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:34,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:34,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:34,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +17: [2023-05-13 09:38:34,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +15: [2023-05-13 09:38:34,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:34,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:34,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:34,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:34,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:34,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +10: [2023-05-13 09:38:34,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:34,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:34,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:34,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:34,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:34,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:34,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:34,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +10: [2023-05-13 09:38:34,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:34,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:34,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:34,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +15: [2023-05-13 09:38:34,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:34,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +28: [2023-05-13 09:38:34,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:34,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +28: [2023-05-13 09:38:34,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:34,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:34,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:34,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:34,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:34,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:34,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:34,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:34,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +31: [2023-05-13 09:38:34,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +31: [2023-05-13 09:38:34,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:34,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +28: [2023-05-13 09:38:34,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +28: [2023-05-13 09:38:34,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +28: [2023-05-13 09:38:34,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +28: [2023-05-13 09:38:34,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:34,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +12: [2023-05-13 09:38:34,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +12: [2023-05-13 09:38:34,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +12: [2023-05-13 09:38:34,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_11-model_01-model_states.pt. +26: [2023-05-13 09:38:34,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:34,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:34,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:34,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:34,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:34,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:34,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:34,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:34,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:34,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:34,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:34,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:34,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:34,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:34,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:34,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:34,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:34,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:34,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:34,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:34,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:34,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:34,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt... +26: [2023-05-13 09:38:34,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +26: [2023-05-13 09:38:34,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:34,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +26: [2023-05-13 09:38:34,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:34,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:34,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:34,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:34,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:34,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:34,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:34,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:34,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:34,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:34,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:34,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:34,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:34,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:34,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:34,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:34,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:34,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:34,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:34,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +25: [2023-05-13 09:38:34,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:34,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:34,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:34,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:34,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:34,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:34,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:34,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:34,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:34,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:34,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +25: [2023-05-13 09:38:34,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:34,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:34,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:34,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:34,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:34,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:34,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:34,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:34,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:34,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:34,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:34,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:34,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:34,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:34,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +24: [2023-05-13 09:38:34,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:34,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:34,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:34,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +24: [2023-05-13 09:38:34,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:34,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:34,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:34,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:34,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +13: [2023-05-13 09:38:34,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:34,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:34,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:34,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +24: [2023-05-13 09:38:34,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:34,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:34,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:34,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +15: [2023-05-13 09:38:34,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:34,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +19: [2023-05-13 09:38:34,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:34,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +19: [2023-05-13 09:38:34,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:34,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:34,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:34,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:34,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:34,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:34,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:34,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +28: [2023-05-13 09:38:34,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:34,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:34,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:34,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:34,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:34,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:34,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:34,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:34,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:34,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:34,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:34,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:34,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:34,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +23: [2023-05-13 09:38:34,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +23: [2023-05-13 09:38:34,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:34,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:34,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:34,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:34,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:34,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:34,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:34,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:34,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:34,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:34,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:34,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:34,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:34,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:34,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:34,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:34,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:34,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:34,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:34,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:34,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:34,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:34,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:34,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:34,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:34,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +21: [2023-05-13 09:38:34,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:34,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +22: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:34,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:34,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:34,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:34,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:34,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:34,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:34,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:34,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:34,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:34,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:34,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:34,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +24: [2023-05-13 09:38:34,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:34,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:34,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:34,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:34,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:34,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:34,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:34,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +19: [2023-05-13 09:38:34,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:34,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:34,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:34,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:34,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:34,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:34,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:34,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:34,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:34,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:34,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +23: [2023-05-13 09:38:34,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +27: [2023-05-13 09:38:34,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:34,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:34,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:34,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:34,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:34,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +15: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:34,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:34,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:34,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:34,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:34,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:34,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:34,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:34,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:34,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:34,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:34,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:34,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:34,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:34,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:34,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:34,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:34,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:34,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:34,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:34,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:34,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:34,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +13: [2023-05-13 09:38:34,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:34,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +21: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:34,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +21: [2023-05-13 09:38:34,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:34,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:34,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:34,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:34,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:34,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:34,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:34,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:34,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:34,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:34,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +13: [2023-05-13 09:38:34,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:34,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:34,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:34,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:34,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +22: [2023-05-13 09:38:34,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:34,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:34,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:34,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:34,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:34,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:34,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +21: [2023-05-13 09:38:34,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:34,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:34,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +17: [2023-05-13 09:38:34,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:34,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:34,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +17: [2023-05-13 09:38:34,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:34,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +15: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:34,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:34,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:34,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:34,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:34,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:34,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:34,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:34,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:34,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +21: [2023-05-13 09:38:34,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:34,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:34,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:34,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +17: [2023-05-13 09:38:34,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:34,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:34,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:34,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +18: [2023-05-13 09:38:34,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:34,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:34,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:34,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:34,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:34,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:34,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:34,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:34,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:34,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +13: [2023-05-13 09:38:34,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +13: [2023-05-13 09:38:34,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:34,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:34,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:34,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:34,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:34,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:34,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:34,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:34,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:34,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:34,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:34,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:34,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +27: [2023-05-13 09:38:34,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:34,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:34,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:34,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:34,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:34,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:34,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:34,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:34,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +14: [2023-05-13 09:38:34,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +13: [2023-05-13 09:38:34,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:34,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:34,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +29: [2023-05-13 09:38:34,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +29: [2023-05-13 09:38:34,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +29: [2023-05-13 09:38:34,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +14: [2023-05-13 09:38:34,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:34,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:34,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +28: [2023-05-13 09:38:34,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +10: [2023-05-13 09:38:34,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:34,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:34,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:34,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:34,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:34,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:34,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_12-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +17: [2023-05-13 09:38:34,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:34,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:34,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:34,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:34,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:34,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:34,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:34,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:34,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:34,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:34,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:34,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:34,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:34,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:34,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:34,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:34,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +27: [2023-05-13 09:38:34,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:34,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:34,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:34,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:34,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:34,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:34,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:34,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:34,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:34,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:34,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:34,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:34,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:34,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:34,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:34,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:34,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:34,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:34,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:34,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:34,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:34,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:34,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:34,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:34,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:34,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:34,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:34,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 6: [2023-05-13 09:38:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 6: [2023-05-13 09:38:34,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 6: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 6: [2023-05-13 09:38:34,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:34,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +27: [2023-05-13 09:38:34,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +31: [2023-05-13 09:38:34,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +31: [2023-05-13 09:38:34,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +31: [2023-05-13 09:38:34,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +23: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:34,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +27: [2023-05-13 09:38:34,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:34,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +23: [2023-05-13 09:38:34,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:34,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:34,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:34,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:34,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +10: [2023-05-13 09:38:34,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:34,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +27: [2023-05-13 09:38:34,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +17: [2023-05-13 09:38:34,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:34,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +27: [2023-05-13 09:38:34,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:34,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +10: [2023-05-13 09:38:34,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:34,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:34,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:34,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:34,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +10: [2023-05-13 09:38:34,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:34,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +10: [2023-05-13 09:38:34,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:34,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:34,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:34,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +10: [2023-05-13 09:38:34,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:34,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:34,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:34,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:34,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:34,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:34,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +17: [2023-05-13 09:38:34,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:34,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +29: [2023-05-13 09:38:34,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +29: [2023-05-13 09:38:34,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:34,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:34,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +17: [2023-05-13 09:38:34,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:34,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:34,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:34,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +17: [2023-05-13 09:38:34,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:34,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:34,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:34,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:34,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:34,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:34,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:34,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:34,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +29: [2023-05-13 09:38:34,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:34,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:34,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:34,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:34,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:34,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:34,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:34,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +30: [2023-05-13 09:38:34,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +30: [2023-05-13 09:38:34,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:34,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:34,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +30: [2023-05-13 09:38:34,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +30: [2023-05-13 09:38:34,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +30: [2023-05-13 09:38:34,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:34,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:34,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:34,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:34,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:34,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:34,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:34,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:34,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:34,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:34,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:34,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:34,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:34,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:34,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:34,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:34,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:34,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:34,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:34,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:34,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:34,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:34,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:34,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:34,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:34,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:34,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:34,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:34,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:34,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:34,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:34,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:34,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:34,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:34,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +24: [2023-05-13 09:38:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +24: [2023-05-13 09:38:34,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +24: [2023-05-13 09:38:34,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:34,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:34,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:34,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:34,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:34,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:34,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:34,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:34,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:34,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:34,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:34,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:34,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:34,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:34,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:34,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:34,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:34,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:34,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:34,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:34,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:34,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:34,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:34,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:34,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:34,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:34,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:34,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +19: [2023-05-13 09:38:34,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +19: [2023-05-13 09:38:34,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +19: [2023-05-13 09:38:34,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +24: [2023-05-13 09:38:34,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:34,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:34,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:34,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:34,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:34,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:34,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:34,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:34,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:34,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:34,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:34,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:34,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:34,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +21: [2023-05-13 09:38:34,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +21: [2023-05-13 09:38:34,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +21: [2023-05-13 09:38:34,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +24: [2023-05-13 09:38:34,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:34,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:34,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:34,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:34,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:34,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:34,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +20: [2023-05-13 09:38:34,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +20: [2023-05-13 09:38:34,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +20: [2023-05-13 09:38:34,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:34,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:34,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:34,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +21: [2023-05-13 09:38:34,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:34,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:34,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:34,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:34,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:34,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:34,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:34,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:34,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:34,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:34,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:34,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:34,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:34,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:34,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:34,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:34,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +16: [2023-05-13 09:38:34,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:34,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +19: [2023-05-13 09:38:34,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:34,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +26: [2023-05-13 09:38:34,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:34,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:34,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +20: [2023-05-13 09:38:34,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:34,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:34,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:34,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:34,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:34,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:34,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:34,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:34,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:34,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:34,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:34,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:34,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:34,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:34,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:34,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:34,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:34,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:34,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:34,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:34,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:34,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:34,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:34,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:34,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:34,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:34,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:34,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +21: [2023-05-13 09:38:34,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:34,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:34,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:34,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:34,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:34,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:34,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:34,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:34,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:34,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:34,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:34,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:34,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:34,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:34,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +21: [2023-05-13 09:38:34,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:34,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:34,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +22: [2023-05-13 09:38:34,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +22: [2023-05-13 09:38:34,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:34,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +20: [2023-05-13 09:38:34,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:34,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +18: [2023-05-13 09:38:34,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:34,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:34,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:34,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:34,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:34,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:34,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:34,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:34,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:34,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:34,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:34,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:34,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:34,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:34,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +23: [2023-05-13 09:38:34,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 9: [2023-05-13 09:38:34,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:34,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +23: [2023-05-13 09:38:34,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:34,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:34,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:34,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:34,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:34,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:34,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:34,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:34,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:34,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:34,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:34,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:34,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:34,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:34,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:34,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:34,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:34,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:34,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:34,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:34,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:34,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:34,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:34,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:34,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:34,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:34,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:34,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:34,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:34,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:34,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:34,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:34,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:34,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:34,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:34,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:34,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:34,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:34,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:34,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:34,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:34,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:34,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:34,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:34,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:34,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:34,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:34,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:34,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:34,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:34,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:34,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +28: [2023-05-13 09:38:34,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:34,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:34,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:34,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:34,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:34,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:34,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +28: [2023-05-13 09:38:34,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:34,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +29: [2023-05-13 09:38:34,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:34,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:34,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:34,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:34,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:34,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:34,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +29: [2023-05-13 09:38:34,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:34,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:34,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:34,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:34,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +23: [2023-05-13 09:38:34,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +28: [2023-05-13 09:38:34,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:34,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:34,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:34,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:34,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:34,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:34,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:34,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:34,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:34,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:34,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:34,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:34,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:34,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:34,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:34,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:34,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:34,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:34,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:34,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:34,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:34,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +23: [2023-05-13 09:38:34,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:34,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:34,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:34,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:34,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:34,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:34,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:34,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:34,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:34,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:34,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:34,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:34,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:34,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +16: [2023-05-13 09:38:34,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:34,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:34,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +11: [2023-05-13 09:38:34,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:34,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:34,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:34,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:34,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:34,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:34,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:34,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:34,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:34,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:34,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:34,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:34,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:34,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:34,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:34,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:34,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:34,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:34,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:34,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:34,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:34,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:34,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:34,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:34,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:34,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:34,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:34,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:34,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:34,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:34,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:34,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:34,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:34,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:34,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:34,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:34,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:34,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:34,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:34,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:34,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:34,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:34,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:34,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:34,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:34,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:34,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:34,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:34,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:34,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:34,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:34,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +12: [2023-05-13 09:38:34,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:34,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +12: [2023-05-13 09:38:34,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:34,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:34,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:34,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:34,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +26: [2023-05-13 09:38:34,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +11: [2023-05-13 09:38:34,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:34,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +26: [2023-05-13 09:38:34,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:34,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +31: [2023-05-13 09:38:34,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:34,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +31: [2023-05-13 09:38:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +12: [2023-05-13 09:38:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:34,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:34,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:34,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:34,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +13: [2023-05-13 09:38:34,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +13: [2023-05-13 09:38:34,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:34,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:34,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +30: [2023-05-13 09:38:34,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +30: [2023-05-13 09:38:34,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +30: [2023-05-13 09:38:34,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:34,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:34,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:34,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:34,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:34,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:34,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:34,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:34,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:34,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:34,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:34,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:34,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:34,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:34,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:34,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:34,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +25: [2023-05-13 09:38:34,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +25: [2023-05-13 09:38:34,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 2: [2023-05-13 09:38:34,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:34,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:34,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:34,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:34,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:34,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:34,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:34,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:34,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:34,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:34,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:34,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:34,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:34,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +19: [2023-05-13 09:38:34,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +13: [2023-05-13 09:38:34,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +16: [2023-05-13 09:38:34,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:34,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:34,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:34,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:34,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:34,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:34,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:34,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:34,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:34,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:34,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:34,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:34,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:34,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:34,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:34,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:34,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:34,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +22: [2023-05-13 09:38:34,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:34,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:34,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +16: [2023-05-13 09:38:34,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:34,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:34,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:34,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:34,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:34,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:34,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:34,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:34,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:34,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:34,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:34,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 6: [2023-05-13 09:38:34,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +15: [2023-05-13 09:38:34,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +12: [2023-05-13 09:38:34,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 6: [2023-05-13 09:38:34,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:34,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:34,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:34,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:34,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:34,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:34,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:34,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:34,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:34,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +27: [2023-05-13 09:38:34,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +27: [2023-05-13 09:38:34,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:34,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:34,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:34,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:34,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:34,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:34,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:34,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:34,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +27: [2023-05-13 09:38:34,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +13: [2023-05-13 09:38:34,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:34,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +31: [2023-05-13 09:38:34,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:34,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:34,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +22: [2023-05-13 09:38:34,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:34,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:34,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:34,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:34,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:34,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:34,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:34,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +31: [2023-05-13 09:38:34,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:34,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +22: [2023-05-13 09:38:34,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:34,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:34,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:34,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:34,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:34,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:34,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:34,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:34,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:34,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:34,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:34,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:34,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:34,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:34,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:34,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:34,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:34,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:34,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 0: [2023-05-13 09:38:34,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:34,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:34,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:34,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +10: [2023-05-13 09:38:34,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +10: [2023-05-13 09:38:34,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:34,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:34,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_13-model_01-model_states.pt. +19: [2023-05-13 09:38:34,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:34,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:34,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:34,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:34,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:34,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:34,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:34,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:34,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:34,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:34,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:34,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 3: [2023-05-13 09:38:34,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +13: [2023-05-13 09:38:34,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:34,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:34,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:34,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:34,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:34,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:34,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +14: [2023-05-13 09:38:34,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:34,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:34,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:34,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:34,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +14: [2023-05-13 09:38:34,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:34,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:34,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:34,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:34,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:34,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 3: [2023-05-13 09:38:34,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +25: [2023-05-13 09:38:34,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:34,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:34,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:34,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:34,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +19: [2023-05-13 09:38:34,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:34,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:34,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:34,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:34,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:34,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:34,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +28: [2023-05-13 09:38:34,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:34,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:34,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:34,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:34,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:34,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:34,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:34,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +28: [2023-05-13 09:38:34,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:34,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +10: [2023-05-13 09:38:34,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:34,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:34,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +28: [2023-05-13 09:38:34,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:34,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:34,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:34,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:34,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:34,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:34,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +30: [2023-05-13 09:38:34,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:34,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:34,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:34,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:34,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:34,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:34,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:34,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +29: [2023-05-13 09:38:34,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:34,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:34,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +22: [2023-05-13 09:38:34,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +22: [2023-05-13 09:38:34,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +22: [2023-05-13 09:38:34,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:34,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:34,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:34,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:34,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:34,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:34,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:34,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:34,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:34,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:34,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:34,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:34,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:34,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:34,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:34,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:34,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:34,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:34,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:34,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:34,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:34,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:34,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:34,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:34,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:34,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:34,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:34,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:34,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:34,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:34,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 6: [2023-05-13 09:38:34,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +20: [2023-05-13 09:38:34,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:34,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:34,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:34,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:34,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:34,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:34,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:34,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:34,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:34,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:34,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:34,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:34,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:34,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:34,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:34,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:34,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +10: [2023-05-13 09:38:34,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:34,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:34,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:34,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:34,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:34,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:34,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:34,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:34,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:34,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:34,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:34,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:34,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:34,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:34,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:34,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:34,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +28: [2023-05-13 09:38:34,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:34,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:34,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:34,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:34,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:34,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:34,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:34,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:34,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:34,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:34,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:34,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:34,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:34,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:34,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:34,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:34,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:34,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:34,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:34,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:34,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +28: [2023-05-13 09:38:34,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:34,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:34,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +20: [2023-05-13 09:38:34,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:34,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:34,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:34,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:34,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:34,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +20: [2023-05-13 09:38:34,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:34,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:34,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:34,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:34,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:34,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:34,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:34,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:34,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:34,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:34,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:34,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:34,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:34,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:34,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:34,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:34,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:34,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:34,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:34,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:34,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:34,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:34,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:34,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:34,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:34,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:34,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +23: [2023-05-13 09:38:34,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:34,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:34,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:34,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:34,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:34,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:34,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:34,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:34,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:34,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:34,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:34,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:34,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:34,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:34,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:34,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:34,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:34,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 9: [2023-05-13 09:38:34,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:34,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:34,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:34,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:34,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:34,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 5: [2023-05-13 09:38:34,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 7: [2023-05-13 09:38:34,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:34,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:34,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:34,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:34,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:34,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:34,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:34,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:34,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 5: [2023-05-13 09:38:34,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 8: [2023-05-13 09:38:34,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:34,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:35,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:35,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:35,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:35,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:35,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:35,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +28: [2023-05-13 09:38:35,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +28: [2023-05-13 09:38:35,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +28: [2023-05-13 09:38:35,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:35,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +18: [2023-05-13 09:38:35,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +18: [2023-05-13 09:38:35,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:35,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +28: [2023-05-13 09:38:35,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:35,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:35,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:35,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:35,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:35,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:35,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:35,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:35,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:35,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:35,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:35,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 7: [2023-05-13 09:38:35,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:35,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +13: [2023-05-13 09:38:35,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +13: [2023-05-13 09:38:35,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:35,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:35,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:35,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:35,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:35,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:35,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:35,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:35,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +27: [2023-05-13 09:38:35,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:35,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:35,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 7: [2023-05-13 09:38:35,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +27: [2023-05-13 09:38:35,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:35,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:35,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:35,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:35,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:35,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:35,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:35,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:35,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:35,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:35,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:35,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:35,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:35,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:35,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:35,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:35,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +11: [2023-05-13 09:38:35,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +11: [2023-05-13 09:38:35,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +11: [2023-05-13 09:38:35,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:35,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:35,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +20: [2023-05-13 09:38:35,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:35,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:35,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +27: [2023-05-13 09:38:35,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:35,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:35,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:35,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +16: [2023-05-13 09:38:35,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:35,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:35,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:35,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +20: [2023-05-13 09:38:35,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:35,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:35,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:35,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:35,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:35,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:35,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:35,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:35,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:35,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:35,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:35,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:35,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:35,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:35,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +16: [2023-05-13 09:38:35,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +16: [2023-05-13 09:38:35,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +16: [2023-05-13 09:38:35,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:35,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:35,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:35,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:35,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:35,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:35,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:35,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:35,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:35,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +18: [2023-05-13 09:38:35,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:35,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +16: [2023-05-13 09:38:35,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:35,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +12: [2023-05-13 09:38:35,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +12: [2023-05-13 09:38:35,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +12: [2023-05-13 09:38:35,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +19: [2023-05-13 09:38:35,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:35,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:35,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:35,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:35,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:35,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:35,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +25: [2023-05-13 09:38:35,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:35,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +25: [2023-05-13 09:38:35,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +25: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +14: [2023-05-13 09:38:35,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:35,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +14: [2023-05-13 09:38:35,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +11: [2023-05-13 09:38:35,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:35,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:35,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:35,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:35,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:35,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:35,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:35,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:35,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:35,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +12: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:35,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +12: [2023-05-13 09:38:35,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:35,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:35,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:35,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:35,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:35,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:35,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:35,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:35,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +25: [2023-05-13 09:38:35,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:35,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:35,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:35,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:35,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +16: [2023-05-13 09:38:35,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +14: [2023-05-13 09:38:35,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:35,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:35,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:35,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:35,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +14: [2023-05-13 09:38:35,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:35,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:35,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:35,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:35,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +30: [2023-05-13 09:38:35,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +30: [2023-05-13 09:38:35,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:35,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +25: [2023-05-13 09:38:35,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:35,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:35,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:35,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +16: [2023-05-13 09:38:35,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +31: [2023-05-13 09:38:35,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:35,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:35,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:35,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:35,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +24: [2023-05-13 09:38:35,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:35,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:35,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:35,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +24: [2023-05-13 09:38:35,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:35,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:35,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:35,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:35,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:35,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:35,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:35,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:35,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +15: [2023-05-13 09:38:35,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:35,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +15: [2023-05-13 09:38:35,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:35,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:35,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:35,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +26: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +26: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +26: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +26: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:35,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +27: [2023-05-13 09:38:35,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:35,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:35,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:35,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:35,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:35,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:35,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:35,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:35,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:35,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:35,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:35,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:35,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:35,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:35,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:35,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:35,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +29: [2023-05-13 09:38:35,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +29: [2023-05-13 09:38:35,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +29: [2023-05-13 09:38:35,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:35,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:35,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:35,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:35,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:35,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:35,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:35,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:35,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:35,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:35,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:35,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:35,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:35,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:35,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:35,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:35,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:35,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:35,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:35,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:35,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:35,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:35,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:35,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:35,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:35,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:35,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:35,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:35,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +14: [2023-05-13 09:38:35,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:35,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:35,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +15: [2023-05-13 09:38:35,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:35,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:35,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:35,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +28: [2023-05-13 09:38:35,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:35,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:35,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:35,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:35,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +14: [2023-05-13 09:38:35,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +14: [2023-05-13 09:38:35,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:35,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:35,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:35,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:35,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:35,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:35,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +25: [2023-05-13 09:38:35,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +29: [2023-05-13 09:38:35,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:35,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:35,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +25: [2023-05-13 09:38:35,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:35,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:35,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:35,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:35,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:35,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:35,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:35,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:35,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +30: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:35,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:35,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:35,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:35,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:35,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:35,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +23: [2023-05-13 09:38:35,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +23: [2023-05-13 09:38:35,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:35,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +24: [2023-05-13 09:38:35,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:35,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:35,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +22: [2023-05-13 09:38:35,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:35,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:35,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +30: [2023-05-13 09:38:35,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:35,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:35,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_14-model_01-model_states.pt. +30: [2023-05-13 09:38:35,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +26: [2023-05-13 09:38:35,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +22: [2023-05-13 09:38:35,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:35,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:35,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:35,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:35,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +12: [2023-05-13 09:38:35,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:35,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:35,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:35,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:35,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:35,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:35,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:35,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +27: [2023-05-13 09:38:35,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:35,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:35,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:35,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:35,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:35,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:35,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:35,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:35,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +14: [2023-05-13 09:38:35,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:35,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:35,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:35,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:35,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:35,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:35,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +26: [2023-05-13 09:38:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:35,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:35,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:35,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:35,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:35,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:35,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:35,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:35,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:35,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:35,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:35,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:35,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:35,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:35,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +21: [2023-05-13 09:38:35,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +21: [2023-05-13 09:38:35,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +21: [2023-05-13 09:38:35,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:35,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:35,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:35,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:35,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:35,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:35,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:35,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:35,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:35,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:35,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +29: [2023-05-13 09:38:35,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +23: [2023-05-13 09:38:35,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:35,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:35,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:35,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:35,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:35,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:35,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:35,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:35,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +22: [2023-05-13 09:38:35,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:35,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:35,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:35,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:35,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:35,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:35,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:35,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:35,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:35,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:35,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:35,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:35,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:35,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:35,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:35,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:35,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:35,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:35,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:35,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +20: [2023-05-13 09:38:35,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:35,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:35,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:35,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:35,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:35,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:35,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:35,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:35,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:35,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:35,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... +14: [2023-05-13 09:38:35,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:35,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:35,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:35,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:35,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:35,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:35,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:35,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:35,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:35,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:35,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:35,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +26: [2023-05-13 09:38:35,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:35,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:35,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:35,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:35,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:35,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:35,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:35,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:35,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:35,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:35,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:35,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:35,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:35,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:35,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:35,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:35,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:35,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:35,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:35,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:35,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:35,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:35,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:35,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +26: [2023-05-13 09:38:35,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:35,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:35,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:35,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:35,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:35,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:35,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:35,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:35,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +23: [2023-05-13 09:38:35,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:35,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:35,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:35,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:35,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:35,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +16: [2023-05-13 09:38:35,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +16: [2023-05-13 09:38:35,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:35,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:35,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +25: [2023-05-13 09:38:35,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +16: [2023-05-13 09:38:35,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:35,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +25: [2023-05-13 09:38:35,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +25: [2023-05-13 09:38:35,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:35,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:35,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:35,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:35,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:35,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:35,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +16: [2023-05-13 09:38:35,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:35,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:35,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:35,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:35,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:35,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:35,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:35,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:35,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:35,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:35,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:35,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +18: [2023-05-13 09:38:35,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:35,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:35,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:35,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:35,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +28: [2023-05-13 09:38:35,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:35,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:35,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +18: [2023-05-13 09:38:35,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +14: [2023-05-13 09:38:35,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +14: [2023-05-13 09:38:35,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +14: [2023-05-13 09:38:35,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +28: [2023-05-13 09:38:35,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:35,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:35,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:35,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:35,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:35,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 7: [2023-05-13 09:38:35,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +19: [2023-05-13 09:38:35,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:35,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:35,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +10: [2023-05-13 09:38:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +10: [2023-05-13 09:38:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:35,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:35,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 5: [2023-05-13 09:38:35,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +18: [2023-05-13 09:38:35,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:35,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:35,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 7: [2023-05-13 09:38:35,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 5: [2023-05-13 09:38:35,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:35,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:35,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:35,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:35,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:35,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:35,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:35,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:35,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:35,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +20: [2023-05-13 09:38:35,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +20: [2023-05-13 09:38:35,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +20: [2023-05-13 09:38:35,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:35,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:35,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:35,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 7: [2023-05-13 09:38:35,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +12: [2023-05-13 09:38:35,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +13: [2023-05-13 09:38:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +18: [2023-05-13 09:38:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:35,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:35,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:35,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:35,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:35,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:35,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:35,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:35,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:35,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:35,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:35,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +25: [2023-05-13 09:38:35,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:35,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:35,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:35,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:35,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:35,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:35,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:35,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:35,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:35,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:35,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:35,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:35,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +30: [2023-05-13 09:38:35,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +30: [2023-05-13 09:38:35,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +30: [2023-05-13 09:38:35,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +20: [2023-05-13 09:38:35,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:35,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:35,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:35,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:35,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:35,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:35,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:35,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:35,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:35,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:35,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:35,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:35,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:35,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:35,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:35,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +30: [2023-05-13 09:38:35,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +11: [2023-05-13 09:38:35,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +13: [2023-05-13 09:38:35,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:35,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:35,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:35,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:35,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:35,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +29: [2023-05-13 09:38:35,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:35,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:35,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:35,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:35,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:35,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:35,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:35,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:35,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:35,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:35,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:35,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:35,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:35,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:35,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:35,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:35,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:35,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:35,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:35,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:35,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:35,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:35,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:35,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:35,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:35,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +11: [2023-05-13 09:38:35,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:35,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +13: [2023-05-13 09:38:35,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:35,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:35,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:35,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:35,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:35,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:35,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:35,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +13: [2023-05-13 09:38:35,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:35,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:35,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:35,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:35,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:35,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:35,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:35,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:35,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:35,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:35,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:35,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:35,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:35,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:35,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:35,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:35,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:35,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:35,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:35,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:35,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:35,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:35,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:35,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:35,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:35,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +15: [2023-05-13 09:38:35,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:35,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:35,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:35,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:35,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:35,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:35,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:35,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:35,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:35,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:35,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:35,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:35,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:35,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:35,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:35,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:35,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:35,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:35,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +25: [2023-05-13 09:38:35,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:35,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:35,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:35,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:35,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:35,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:35,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:35,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:35,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:35,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:35,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:35,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:35,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:35,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:35,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:35,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:35,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:35,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:35,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:35,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:35,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:35,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:35,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:35,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:35,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +19: [2023-05-13 09:38:35,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +26: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:35,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +27: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +27: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +19: [2023-05-13 09:38:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +27: [2023-05-13 09:38:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +26: [2023-05-13 09:38:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +26: [2023-05-13 09:38:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:35,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:35,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:35,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:35,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:35,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:35,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +24: [2023-05-13 09:38:35,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:35,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:35,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:35,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:35,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:35,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:35,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:35,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +23: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +23: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +23: [2023-05-13 09:38:35,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:35,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:35,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +27: [2023-05-13 09:38:35,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:35,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:35,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:35,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:35,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:35,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:35,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:35,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:35,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:35,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:35,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:35,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:35,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:35,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:35,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +25: [2023-05-13 09:38:35,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:35,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:35,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:35,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:35,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:35,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:35,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:35,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:35,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:35,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +21: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:35,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:35,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:35,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:35,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:35,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:35,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:35,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +27: [2023-05-13 09:38:35,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:35,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:35,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +20: [2023-05-13 09:38:35,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:35,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +21: [2023-05-13 09:38:35,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:35,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:35,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:35,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +23: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:35,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:35,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:35,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:35,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +24: [2023-05-13 09:38:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:35,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:35,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +23: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +24: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:35,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:35,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:35,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:35,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:35,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +21: [2023-05-13 09:38:35,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +29: [2023-05-13 09:38:35,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +29: [2023-05-13 09:38:35,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +29: [2023-05-13 09:38:35,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +21: [2023-05-13 09:38:35,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:35,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 5: [2023-05-13 09:38:35,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +20: [2023-05-13 09:38:35,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:35,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:35,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:35,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:35,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:35,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:35,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:35,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:35,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:35,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:35,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:35,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:35,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:35,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:35,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:35,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:35,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:35,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:35,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +26: [2023-05-13 09:38:35,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:35,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:35,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:35,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:35,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:35,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:35,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:35,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +13: [2023-05-13 09:38:35,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +13: [2023-05-13 09:38:35,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:35,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:35,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:35,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:35,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:35,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:35,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:35,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:35,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:35,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +22: [2023-05-13 09:38:35,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +22: [2023-05-13 09:38:35,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +22: [2023-05-13 09:38:35,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:35,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:35,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +15: [2023-05-13 09:38:35,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +15: [2023-05-13 09:38:35,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +23: [2023-05-13 09:38:35,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:35,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_15-model_01-model_states.pt. +26: [2023-05-13 09:38:35,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:35,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:35,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:35,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +31: [2023-05-13 09:38:35,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:35,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:35,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:35,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:35,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:35,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:35,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:35,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:35,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:35,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:35,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:35,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:35,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:35,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:35,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:35,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:35,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:35,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:35,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:35,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:35,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:35,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +23: [2023-05-13 09:38:35,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:35,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:35,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:35,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:35,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:35,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:35,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +18: [2023-05-13 09:38:35,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:35,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:35,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:35,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:35,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:35,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:35,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +26: [2023-05-13 09:38:35,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:35,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:35,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:35,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:35,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:35,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:35,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +21: [2023-05-13 09:38:35,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:35,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:35,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:35,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:35,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:35,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:35,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +22: [2023-05-13 09:38:35,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:35,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:35,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:35,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +22: [2023-05-13 09:38:35,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:35,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:35,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:35,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:35,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:35,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:35,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:35,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:35,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:35,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:35,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:35,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:35,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:35,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:35,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:35,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:35,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:35,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:35,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:35,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:35,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:35,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:35,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:35,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:35,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:35,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:35,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:35,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:35,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:35,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:35,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:35,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:35,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:35,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:35,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:35,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:35,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:35,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:35,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:35,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +19: [2023-05-13 09:38:35,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:35,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:35,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:35,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:35,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:35,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:35,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:35,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:35,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:35,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:35,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:35,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:35,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:35,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:35,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:35,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:35,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:35,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:35,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:35,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:35,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:35,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:35,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +10: [2023-05-13 09:38:35,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +10: [2023-05-13 09:38:35,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +10: [2023-05-13 09:38:35,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +15: [2023-05-13 09:38:35,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +27: [2023-05-13 09:38:35,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +27: [2023-05-13 09:38:35,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:35,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:35,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:35,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:35,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:35,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:35,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:35,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:35,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:35,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:35,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:35,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:35,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:35,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:35,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:35,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +12: [2023-05-13 09:38:35,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +16: [2023-05-13 09:38:35,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:35,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:35,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:35,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:35,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:35,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:35,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:35,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:35,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:35,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:35,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:35,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:35,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:35,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:35,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:35,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:35,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:35,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +29: [2023-05-13 09:38:35,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +29: [2023-05-13 09:38:35,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +29: [2023-05-13 09:38:35,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:35,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:35,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:35,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:35,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +19: [2023-05-13 09:38:35,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:35,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:35,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:35,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:35,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:35,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:35,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:35,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:35,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:35,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:35,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:35,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:35,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:35,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:35,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:35,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:35,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:35,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:35,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:35,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:35,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:35,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:35,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:35,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:35,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:35,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:35,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:35,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:35,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:35,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:35,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:35,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:35,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:35,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:35,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:35,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:35,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:35,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:35,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:35,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:35,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:35,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:35,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +20: [2023-05-13 09:38:35,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +20: [2023-05-13 09:38:35,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:35,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +29: [2023-05-13 09:38:35,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:35,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:35,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:35,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:35,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:35,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:35,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:35,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:35,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:35,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:35,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:35,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:35,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:35,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 0: [2023-05-13 09:38:35,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:35,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:35,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +14: [2023-05-13 09:38:35,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +14: [2023-05-13 09:38:35,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +14: [2023-05-13 09:38:35,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:35,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:35,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:35,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:35,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:35,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:35,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:35,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:35,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:35,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:35,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:35,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:35,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:35,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:35,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:35,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:35,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:35,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:35,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:35,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +12: [2023-05-13 09:38:35,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:35,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:35,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:35,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:35,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:35,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:35,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:35,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +20: [2023-05-13 09:38:35,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +13: [2023-05-13 09:38:35,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:35,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:35,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:35,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:35,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:35,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:35,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:35,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:35,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:35,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:35,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:35,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:35,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:35,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:35,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:35,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:35,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:35,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:35,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:35,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:35,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:35,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:35,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:35,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:35,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:35,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:35,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:35,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:35,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:35,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:35,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:35,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:35,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:35,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:35,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:35,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:35,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:35,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:35,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:35,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +31: [2023-05-13 09:38:35,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +31: [2023-05-13 09:38:35,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +30: [2023-05-13 09:38:35,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:35,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:35,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:35,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:35,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:35,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:35,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:35,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:35,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:35,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:35,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:35,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:35,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:35,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:35,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:35,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:35,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:35,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:35,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:35,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:35,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:35,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:35,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:35,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:35,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:35,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:35,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:35,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:35,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:35,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:35,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:35,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:35,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:35,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:35,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:35,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:35,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:35,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:35,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:35,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +23: [2023-05-13 09:38:35,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:35,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:35,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:35,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:35,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:35,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:35,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:35,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:35,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:35,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:35,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:35,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:35,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:35,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:35,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:35,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:35,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:35,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:35,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +16: [2023-05-13 09:38:35,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:35,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:35,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:35,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:35,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:35,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:35,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:35,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:35,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:35,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:35,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:35,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:35,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:35,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +24: [2023-05-13 09:38:35,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:35,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:35,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +27: [2023-05-13 09:38:35,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:35,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +12: [2023-05-13 09:38:35,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:35,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:35,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +18: [2023-05-13 09:38:35,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +18: [2023-05-13 09:38:35,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:35,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +24: [2023-05-13 09:38:35,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:35,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +24: [2023-05-13 09:38:35,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +18: [2023-05-13 09:38:35,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +18: [2023-05-13 09:38:35,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:35,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:35,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +19: [2023-05-13 09:38:35,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +27: [2023-05-13 09:38:35,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:35,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:35,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +22: [2023-05-13 09:38:35,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +22: [2023-05-13 09:38:35,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:35,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:35,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:35,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:35,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:35,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +23: [2023-05-13 09:38:35,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:35,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:35,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:35,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:35,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +22: [2023-05-13 09:38:35,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:35,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:35,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +23: [2023-05-13 09:38:35,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:35,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:35,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:35,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:35,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:35,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:35,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:35,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:35,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:35,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:35,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:35,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:35,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +22: [2023-05-13 09:38:35,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:35,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:35,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +15: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +16: [2023-05-13 09:38:35,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:35,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:35,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +30: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +30: [2023-05-13 09:38:35,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:35,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +27: [2023-05-13 09:38:35,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +18: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:35,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:35,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:35,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:35,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:35,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:35,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:35,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:35,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:35,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:35,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:35,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:35,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:35,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:35,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:35,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +19: [2023-05-13 09:38:35,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:35,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +13: [2023-05-13 09:38:35,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:35,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:35,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:35,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:35,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:35,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:35,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +13: [2023-05-13 09:38:35,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +23: [2023-05-13 09:38:35,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:35,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:35,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 5: [2023-05-13 09:38:35,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +23: [2023-05-13 09:38:35,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:35,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:35,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +15: [2023-05-13 09:38:35,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:35,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:35,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:35,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:35,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:35,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:35,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:35,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:35,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:35,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:35,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:35,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:35,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:35,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:35,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:35,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:35,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:35,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:35,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:35,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 8: [2023-05-13 09:38:35,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +23: [2023-05-13 09:38:35,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:35,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:35,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:35,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:35,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:35,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:35,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:35,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:35,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:35,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:35,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:35,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +21: [2023-05-13 09:38:35,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:35,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 7: [2023-05-13 09:38:35,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:35,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +21: [2023-05-13 09:38:35,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +13: [2023-05-13 09:38:35,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:35,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:35,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:35,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:35,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:35,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:35,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:35,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:35,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:35,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:35,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:35,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:35,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:35,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +22: [2023-05-13 09:38:35,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:35,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:35,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:35,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:35,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:35,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 7: [2023-05-13 09:38:35,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +19: [2023-05-13 09:38:35,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:35,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:35,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +19: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:35,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:35,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:35,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:35,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:35,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 1: [2023-05-13 09:38:35,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +18: [2023-05-13 09:38:35,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:35,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +19: [2023-05-13 09:38:35,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:35,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:35,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:35,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:35,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:35,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:35,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:35,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:35,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +26: [2023-05-13 09:38:35,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:35,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:35,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:35,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:35,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:35,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:35,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +25: [2023-05-13 09:38:35,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:35,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:35,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +26: [2023-05-13 09:38:35,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:35,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:35,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:35,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:35,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:35,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +23: [2023-05-13 09:38:35,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:35,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +22: [2023-05-13 09:38:35,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:35,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:35,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:35,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:35,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:35,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:35,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:35,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:35,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:35,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:35,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:35,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:35,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:35,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:35,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:35,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:35,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:35,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:35,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:35,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:35,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +25: [2023-05-13 09:38:35,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:35,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +15: [2023-05-13 09:38:35,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:35,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:35,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:35,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:35,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +22: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:35,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:35,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:35,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:35,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:35,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:35,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +13: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +13: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:35,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:35,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +13: [2023-05-13 09:38:35,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:35,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:35,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 6: [2023-05-13 09:38:35,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:35,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +13: [2023-05-13 09:38:35,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:35,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:35,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:35,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:35,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:35,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:35,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +26: [2023-05-13 09:38:35,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:35,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt... +24: [2023-05-13 09:38:35,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:35,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:35,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:35,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:35,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:35,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:35,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:35,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:35,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:35,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:35,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:35,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:35,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:35,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:35,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:35,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:35,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:35,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:35,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:35,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:35,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:35,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:35,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:35,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:35,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:35,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:35,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:35,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:35,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:35,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:35,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:35,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 1: [2023-05-13 09:38:35,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 9: [2023-05-13 09:38:35,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:35,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:35,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:35,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 8: [2023-05-13 09:38:35,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:35,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:35,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:35,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:35,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:35,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:35,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:35,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:35,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:35,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:35,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:35,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:35,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:35,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:35,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:35,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:35,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +14: [2023-05-13 09:38:35,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:35,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:35,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:35,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:35,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +27: [2023-05-13 09:38:35,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +27: [2023-05-13 09:38:35,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +27: [2023-05-13 09:38:35,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:35,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:35,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:35,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:35,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:35,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:35,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:35,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:35,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:35,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:35,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:35,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:35,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:35,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:35,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:35,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:35,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:35,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:35,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:36,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:36,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:36,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:36,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:36,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:36,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:36,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:36,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:36,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:36,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:36,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:36,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:36,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:36,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:36,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:36,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:36,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:36,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:36,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:36,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:36,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:36,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:36,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:36,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:36,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:36,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:36,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:36,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:36,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:36,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:36,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +17: [2023-05-13 09:38:36,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:36,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:36,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:36,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:36,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:36,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:36,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:36,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:36,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:36,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 4: [2023-05-13 09:38:36,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:36,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:36,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:36,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +17: [2023-05-13 09:38:36,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:36,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:36,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:36,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +27: [2023-05-13 09:38:36,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:36,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:36,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:36,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:36,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +16: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:36,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +20: [2023-05-13 09:38:36,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:36,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:36,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:36,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:36,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:36,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:36,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:36,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:36,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:36,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:36,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:36,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:36,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:36,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:36,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:36,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:36,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:36,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:36,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:36,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:36,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +20: [2023-05-13 09:38:36,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +19: [2023-05-13 09:38:36,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:36,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +11: [2023-05-13 09:38:36,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +20: [2023-05-13 09:38:36,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +20: [2023-05-13 09:38:36,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:36,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:36,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +29: [2023-05-13 09:38:36,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +29: [2023-05-13 09:38:36,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +29: [2023-05-13 09:38:36,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:36,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:36,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:36,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +15: [2023-05-13 09:38:36,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:36,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:36,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +15: [2023-05-13 09:38:36,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:36,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:36,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +16: [2023-05-13 09:38:36,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:36,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:36,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:36,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:36,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:36,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:36,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:36,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:36,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:36,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:36,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:36,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:36,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:36,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:36,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:36,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:36,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:36,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:36,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:36,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:36,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:36,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:36,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:36,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:36,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:36,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:36,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:36,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +14: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:36,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:36,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:36,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:36,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:36,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +13: [2023-05-13 09:38:36,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:36,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:36,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:36,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:36,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:36,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:36,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:36,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:36,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:36,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:36,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:36,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:36,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:36,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:36,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +22: [2023-05-13 09:38:36,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:36,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:36,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:36,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:36,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:36,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:36,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:36,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:36,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:36,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:36,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:36,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:36,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:36,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +20: [2023-05-13 09:38:36,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 4: [2023-05-13 09:38:36,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:36,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:36,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:36,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:36,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:36,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:36,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:36,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:36,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:36,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +27: [2023-05-13 09:38:36,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:36,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:36,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:36,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:36,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:36,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:36,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:36,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:36,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:36,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:36,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:36,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:36,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:36,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:36,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +31: [2023-05-13 09:38:36,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:36,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +31: [2023-05-13 09:38:36,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:36,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:36,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:36,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:36,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:36,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:36,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +13: [2023-05-13 09:38:36,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:36,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:36,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:36,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:36,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:36,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:36,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:36,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:36,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:36,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:36,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:36,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +31: [2023-05-13 09:38:36,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:36,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:36,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:36,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:36,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:36,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:36,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:36,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:36,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:36,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:36,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:36,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:36,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:36,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:36,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:36,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:36,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:36,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:36,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:36,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:36,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:36,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:36,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:36,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:36,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:36,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:36,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:36,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:36,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:36,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:36,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:36,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:36,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:36,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:36,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:36,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:36,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:36,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:36,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:36,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:36,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:36,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:36,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:36,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:36,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:36,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:36,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... + 7: [2023-05-13 09:38:36,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:36,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:36,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +14: [2023-05-13 09:38:36,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:36,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:36,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:36,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:36,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:36,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:36,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:36,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:36,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:36,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:36,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:36,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:36,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:36,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:36,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:36,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:36,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:36,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:36,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +14: [2023-05-13 09:38:36,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +11: [2023-05-13 09:38:36,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:36,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:36,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:36,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:36,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:36,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:36,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:36,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:36,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:36,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:36,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:36,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:36,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:36,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:36,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt... +26: [2023-05-13 09:38:36,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:36,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:36,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:36,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:36,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:36,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:36,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:36,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:36,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:36,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:36,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:36,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:36,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:36,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:36,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:36,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:36,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:36,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:36,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:36,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:36,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:36,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:36,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:36,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:36,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:36,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:36,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:36,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:36,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:36,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:36,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:36,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:36,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:36,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:36,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:36,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:36,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:36,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:36,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:36,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:36,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:36,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:36,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:36,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:36,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:36,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:36,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:36,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:36,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:36,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:36,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:36,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:36,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:36,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:36,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:36,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:36,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:36,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:36,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:36,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:36,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:36,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:36,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:36,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:36,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:36,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:36,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:36,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:36,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:36,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:36,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:36,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:36,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:36,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:36,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:36,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:36,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:36,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:36,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:36,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:36,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:36,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:36,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:36,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:36,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:36,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:36,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:36,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:36,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:36,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:36,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:36,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:36,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:36,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:36,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:36,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:36,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:36,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:36,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:36,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:36,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:36,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:36,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:36,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:36,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:36,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:36,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:36,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:36,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:36,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:36,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:36,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:36,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:36,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:36,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:36,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:36,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:36,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:36,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:36,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:36,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:36,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:36,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:36,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:36,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:36,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:36,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:36,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:36,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:36,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:36,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:36,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:36,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:36,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:36,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:36,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:36,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:36,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:36,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:36,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:36,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:36,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:36,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:36,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:36,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:36,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:36,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:36,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:36,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:36,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:36,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:36,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:36,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:36,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:36,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:36,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:36,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:36,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:36,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:36,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:36,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:36,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:36,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:36,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:36,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:36,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:36,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:36,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:36,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:36,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:36,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:36,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:36,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:36,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:36,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:36,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:36,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:36,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:36,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:36,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:36,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:36,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:36,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:36,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:36,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:36,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:36,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:36,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:36,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:36,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:36,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:36,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:36,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:36,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:36,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:36,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:36,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:36,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +21: [2023-05-13 09:38:36,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:36,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:36,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:36,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:36,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:36,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:36,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:36,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:36,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:36,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:36,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:36,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:36,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:36,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:36,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:36,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:36,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:36,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:36,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:36,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:36,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:36,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:36,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:36,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:36,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:36,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:36,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:36,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +16: [2023-05-13 09:38:36,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +16: [2023-05-13 09:38:36,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +16: [2023-05-13 09:38:36,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:36,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:36,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:36,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:36,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:36,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:36,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:36,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:36,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:36,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:36,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:36,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:36,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:36,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:36,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:36,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:36,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:36,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:36,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:36,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:36,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:36,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:36,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:36,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:36,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:36,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:36,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:36,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:36,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:36,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:36,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:36,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:36,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:36,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:36,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:36,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:36,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:36,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:36,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:36,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:36,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:36,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:36,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:36,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:36,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:36,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:36,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:36,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:36,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:36,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:36,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:36,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:36,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:36,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 7: [2023-05-13 09:38:36,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:36,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:36,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:36,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:36,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:36,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:36,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:36,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:36,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:36,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:36,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 4: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:36,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 1: [2023-05-13 09:38:36,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:36,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:36,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:36,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:36,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:36,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:36,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:36,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:36,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:36,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:36,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:36,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:36,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:36,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:36,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:36,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:36,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:36,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:36,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:36,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:36,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:36,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:36,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:36,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:36,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:36,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:36,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:36,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:36,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +19: [2023-05-13 09:38:36,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +11: [2023-05-13 09:38:36,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:36,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +19: [2023-05-13 09:38:36,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:36,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:36,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:36,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:36,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:36,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:36,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:36,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +18: [2023-05-13 09:38:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +18: [2023-05-13 09:38:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +18: [2023-05-13 09:38:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +27: [2023-05-13 09:38:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:36,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:36,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +27: [2023-05-13 09:38:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:36,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:36,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +27: [2023-05-13 09:38:36,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:36,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:36,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:36,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:36,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:36,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:36,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:36,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:36,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:36,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:36,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:36,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:36,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:36,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +19: [2023-05-13 09:38:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:36,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:36,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:36,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:36,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:36,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:36,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:36,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +17: [2023-05-13 09:38:36,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:36,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:36,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:36,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:36,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:36,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 6: [2023-05-13 09:38:36,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:36,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:36,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:36,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:36,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:36,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:36,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:36,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:36,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:36,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +24: [2023-05-13 09:38:36,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +13: [2023-05-13 09:38:36,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:36,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +24: [2023-05-13 09:38:36,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:36,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +13: [2023-05-13 09:38:36,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:36,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:36,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:36,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:36,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 1: [2023-05-13 09:38:36,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 1: [2023-05-13 09:38:36,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +28: [2023-05-13 09:38:36,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:36,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +13: [2023-05-13 09:38:36,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:36,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:36,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:36,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:36,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:36,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:36,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:36,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:36,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +13: [2023-05-13 09:38:36,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:36,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +30: [2023-05-13 09:38:36,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +30: [2023-05-13 09:38:36,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:36,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:36,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:36,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:36,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:36,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:36,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:36,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:36,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +27: [2023-05-13 09:38:36,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:36,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:36,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:36,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:36,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +24: [2023-05-13 09:38:36,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:36,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:36,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:36,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:36,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:36,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:36,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +23: [2023-05-13 09:38:36,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:36,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:36,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:36,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:36,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:36,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:36,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:36,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +30: [2023-05-13 09:38:36,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:36,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:36,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:36,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:36,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:36,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:36,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:36,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:36,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:36,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:36,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +16: [2023-05-13 09:38:36,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:36,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:36,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +27: [2023-05-13 09:38:36,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:36,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:36,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +15: [2023-05-13 09:38:36,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:36,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:36,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:36,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:36,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:36,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:36,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 4: [2023-05-13 09:38:36,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +23: [2023-05-13 09:38:36,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:36,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:36,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +26: [2023-05-13 09:38:36,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 4: [2023-05-13 09:38:36,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:36,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:36,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:36,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:36,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +26: [2023-05-13 09:38:36,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 3: [2023-05-13 09:38:36,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:36,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +24: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:36,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:36,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:36,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:36,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:36,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:36,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +16: [2023-05-13 09:38:36,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:36,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:36,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:36,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:36,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:36,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:36,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:36,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:36,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:36,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:36,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:36,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:36,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:36,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:36,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:36,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:36,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:36,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:36,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:36,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:36,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:36,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 0: [2023-05-13 09:38:36,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:36,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:36,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 8: [2023-05-13 09:38:36,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:36,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:36,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +10: [2023-05-13 09:38:36,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:36,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:36,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:36,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:36,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:36,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:36,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:36,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:36,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:36,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:36,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +27: [2023-05-13 09:38:36,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:36,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:36,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +19: [2023-05-13 09:38:36,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +27: [2023-05-13 09:38:36,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +27: [2023-05-13 09:38:36,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:36,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:36,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:36,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:36,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:36,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:36,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:36,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:36,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:36,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:36,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:36,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:36,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:36,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +23: [2023-05-13 09:38:36,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +23: [2023-05-13 09:38:36,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +21: [2023-05-13 09:38:36,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +21: [2023-05-13 09:38:36,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 8: [2023-05-13 09:38:36,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:36,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +23: [2023-05-13 09:38:36,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +21: [2023-05-13 09:38:36,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +23: [2023-05-13 09:38:36,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:36,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:36,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:36,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:36,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +15: [2023-05-13 09:38:36,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:36,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:36,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:36,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:36,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:36,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:36,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:36,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:36,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:36,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 4: [2023-05-13 09:38:36,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:36,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +15: [2023-05-13 09:38:36,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:36,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:36,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +19: [2023-05-13 09:38:36,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:36,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:36,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:36,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:36,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:36,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:36,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:36,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:36,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:36,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:36,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:36,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:36,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:36,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:36,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:36,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:36,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:36,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:36,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:36,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:36,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:36,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:36,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:36,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:36,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:36,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:36,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:36,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:36,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:36,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:36,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:36,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:36,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:36,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +20: [2023-05-13 09:38:36,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +20: [2023-05-13 09:38:36,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +20: [2023-05-13 09:38:36,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +20: [2023-05-13 09:38:36,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:36,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:36,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:36,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:36,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:36,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:36,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:36,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:36,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:36,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:36,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:36,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:36,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:36,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:36,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:36,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:36,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:36,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +31: [2023-05-13 09:38:36,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +31: [2023-05-13 09:38:36,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:36,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +11: [2023-05-13 09:38:36,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +11: [2023-05-13 09:38:36,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +11: [2023-05-13 09:38:36,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +11: [2023-05-13 09:38:36,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +26: [2023-05-13 09:38:36,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:36,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:36,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +12: [2023-05-13 09:38:36,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:36,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +12: [2023-05-13 09:38:36,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +26: [2023-05-13 09:38:36,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:36,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:36,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:36,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +22: [2023-05-13 09:38:36,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +12: [2023-05-13 09:38:36,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:36,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +29: [2023-05-13 09:38:36,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +29: [2023-05-13 09:38:36,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +29: [2023-05-13 09:38:36,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +12: [2023-05-13 09:38:36,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:36,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:36,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:36,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +22: [2023-05-13 09:38:36,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +20: [2023-05-13 09:38:36,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:36,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:36,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:36,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:36,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:36,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:36,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:36,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:36,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:36,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:36,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:36,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:36,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:36,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:36,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:36,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:36,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:36,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:36,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:36,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:36,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:36,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:36,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:36,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:36,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:36,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:36,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:36,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:36,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:36,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:36,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:36,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:36,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:36,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:36,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 9: [2023-05-13 09:38:36,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:36,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:36,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:36,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:36,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:36,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:36,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:36,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:36,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:36,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:36,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:36,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:36,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:36,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:36,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:36,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:36,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:36,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:36,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +17: [2023-05-13 09:38:36,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +17: [2023-05-13 09:38:36,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +17: [2023-05-13 09:38:36,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:36,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:36,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:36,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:36,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:36,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:36,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 7: [2023-05-13 09:38:36,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:36,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +10: [2023-05-13 09:38:36,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +10: [2023-05-13 09:38:36,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:36,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:36,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:36,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:36,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:36,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:36,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:36,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:36,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:36,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:36,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:36,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:36,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:36,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:36,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:36,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:36,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:36,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:36,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:36,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:36,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:36,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:36,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:36,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:36,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:36,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:36,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:36,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:36,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:36,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:36,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:36,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:36,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:36,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:36,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:36,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:36,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:36,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:36,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:36,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:36,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:36,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +11: [2023-05-13 09:38:36,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +11: [2023-05-13 09:38:36,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:36,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:36,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:36,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:36,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:36,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:36,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:36,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:36,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:36,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:36,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:36,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:36,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:36,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:36,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:36,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:36,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:36,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +11: [2023-05-13 09:38:36,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +20: [2023-05-13 09:38:36,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:36,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:36,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:36,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:36,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:36,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:36,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +11: [2023-05-13 09:38:36,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:36,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:36,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:36,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +13: [2023-05-13 09:38:36,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:36,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:36,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:36,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:36,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:36,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:36,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:36,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:36,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:36,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:36,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:36,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:36,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:36,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:36,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:36,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:36,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:36,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:36,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:36,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +17: [2023-05-13 09:38:36,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:36,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:36,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:36,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:36,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:36,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:36,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:36,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:36,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:36,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:36,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:36,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +17: [2023-05-13 09:38:36,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:36,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +10: [2023-05-13 09:38:36,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 9: [2023-05-13 09:38:36,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:36,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:36,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:36,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:36,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:36,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:36,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:36,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:36,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:36,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:36,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:36,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:36,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:36,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:36,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:36,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:36,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +10: [2023-05-13 09:38:36,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:36,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:36,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:36,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +10: [2023-05-13 09:38:36,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:36,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:36,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:36,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:36,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:36,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:36,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:36,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:36,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:36,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:36,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:36,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:36,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:36,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:36,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:36,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:36,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:36,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:36,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:36,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:36,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:36,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:36,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:36,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:36,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:36,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:36,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:36,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:36,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:36,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:36,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:36,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:36,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:36,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:36,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:36,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:36,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:36,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:36,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:36,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:36,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:36,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:36,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:36,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:36,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:36,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:36,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:36,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:36,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:36,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:36,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:36,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:36,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:36,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:36,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:36,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:36,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:36,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:36,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:36,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:36,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:36,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:36,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:36,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:36,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:36,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:36,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:36,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:36,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:36,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:36,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:36,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:36,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:36,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:36,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:36,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:36,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:36,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:36,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:36,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:36,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:36,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:36,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:36,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:36,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:36,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:36,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:36,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:36,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:36,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:36,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:36,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:36,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:36,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:36,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:36,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:36,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:36,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:36,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:36,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:36,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:36,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:36,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +16: [2023-05-13 09:38:36,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +16: [2023-05-13 09:38:36,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +16: [2023-05-13 09:38:36,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:36,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:36,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:36,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +25: [2023-05-13 09:38:36,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:36,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:36,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:36,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 1: [2023-05-13 09:38:36,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +16: [2023-05-13 09:38:36,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:36,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:36,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:36,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +25: [2023-05-13 09:38:36,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:36,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:36,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:36,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:36,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:36,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:36,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:36,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:36,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +21: [2023-05-13 09:38:36,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:36,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:36,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:36,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:36,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:36,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:36,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:36,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:36,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:36,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:36,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:36,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:36,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:36,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:36,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +18: [2023-05-13 09:38:36,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:36,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:36,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:36,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:36,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:36,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:36,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:36,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:36,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +19: [2023-05-13 09:38:36,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +21: [2023-05-13 09:38:36,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:36,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +19: [2023-05-13 09:38:36,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +16: [2023-05-13 09:38:36,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:36,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:36,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:36,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +21: [2023-05-13 09:38:36,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:36,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:36,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:37,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:37,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +18: [2023-05-13 09:38:37,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:37,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:37,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:37,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:37,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:37,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:37,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:37,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:37,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:37,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:37,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:37,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:37,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:37,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:37,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:37,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:37,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:37,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:37,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:37,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:37,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:37,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:37,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:37,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:37,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:37,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:37,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:37,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:37,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:37,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:37,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:37,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +24: [2023-05-13 09:38:37,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:37,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +12: [2023-05-13 09:38:37,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:37,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:37,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:37,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:37,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:37,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:37,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:37,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:37,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:37,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:37,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:37,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:37,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:37,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:37,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:37,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:37,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:37,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:37,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:37,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:37,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:37,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:37,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:37,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:37,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:37,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:37,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +26: [2023-05-13 09:38:37,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:37,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:37,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:37,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:37,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:37,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:37,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:37,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:37,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:37,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:37,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:37,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:37,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +18: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +18: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +18: [2023-05-13 09:38:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +19: [2023-05-13 09:38:37,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:37,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:37,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:37,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:37,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:37,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:37,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:37,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:37,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:37,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:37,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +26: [2023-05-13 09:38:37,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:37,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:37,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:37,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:37,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:37,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +19: [2023-05-13 09:38:37,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:37,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:37,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:37,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:37,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:37,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:37,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:37,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:37,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:37,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:37,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:37,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:37,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:37,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:37,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:37,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:37,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:37,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:37,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:37,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:37,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:37,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:37,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:37,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:37,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:37,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:37,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:37,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:37,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:37,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +30: [2023-05-13 09:38:37,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +30: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +26: [2023-05-13 09:38:37,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +26: [2023-05-13 09:38:37,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:37,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:37,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:37,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +26: [2023-05-13 09:38:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +26: [2023-05-13 09:38:37,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:37,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:37,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:37,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:37,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:37,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:37,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:37,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:37,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:37,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:37,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +20: [2023-05-13 09:38:37,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:37,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:37,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:37,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:37,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:37,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:37,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:37,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:37,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:37,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:37,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:37,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:37,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:37,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:37,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +27: [2023-05-13 09:38:37,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +27: [2023-05-13 09:38:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +26: [2023-05-13 09:38:37,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:37,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:37,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:37,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:37,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:37,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +19: [2023-05-13 09:38:37,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:37,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +18: [2023-05-13 09:38:37,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +10: [2023-05-13 09:38:37,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:37,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +14: [2023-05-13 09:38:37,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +10: [2023-05-13 09:38:37,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:37,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:37,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:37,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +24: [2023-05-13 09:38:37,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +10: [2023-05-13 09:38:37,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:37,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:37,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:37,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:37,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +15: [2023-05-13 09:38:37,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +15: [2023-05-13 09:38:37,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:37,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:37,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:37,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:37,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +15: [2023-05-13 09:38:37,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:37,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:37,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:37,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:37,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:37,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:37,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:37,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:37,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +10: [2023-05-13 09:38:37,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:37,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:37,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:37,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:37,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +24: [2023-05-13 09:38:37,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:37,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:37,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:37,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:37,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:37,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:37,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:37,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:37,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:37,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:37,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:37,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:37,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:37,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:37,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:37,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:37,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +26: [2023-05-13 09:38:37,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:37,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:37,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:37,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:37,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +20: [2023-05-13 09:38:37,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:37,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +26: [2023-05-13 09:38:37,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +26: [2023-05-13 09:38:37,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +15: [2023-05-13 09:38:37,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +26: [2023-05-13 09:38:37,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +20: [2023-05-13 09:38:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:37,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:37,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +26: [2023-05-13 09:38:37,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:37,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:37,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:37,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:37,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:37,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:37,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:37,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:37,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:37,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:37,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:37,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:37,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +27: [2023-05-13 09:38:37,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +23: [2023-05-13 09:38:37,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:37,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +22: [2023-05-13 09:38:37,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:37,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:37,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:37,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:37,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:37,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +27: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:37,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:37,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:37,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +17: [2023-05-13 09:38:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:37,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:37,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:37,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:37,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:37,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:37,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:37,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:37,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:37,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:37,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:37,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:37,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:37,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:37,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:37,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:37,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:37,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:37,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:37,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:37,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:37,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:37,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:37,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:37,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:37,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:37,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +21: [2023-05-13 09:38:37,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:37,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +21: [2023-05-13 09:38:37,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:37,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:37,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +17: [2023-05-13 09:38:37,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:37,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:37,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:37,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:37,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:37,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:37,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:37,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:37,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:37,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:37,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:37,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:37,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:37,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:37,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:37,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:37,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:37,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:37,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:37,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:37,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +14: [2023-05-13 09:38:37,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:37,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:37,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +10: [2023-05-13 09:38:37,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +10: [2023-05-13 09:38:37,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:37,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:37,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:37,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:37,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +23: [2023-05-13 09:38:37,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:37,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +21: [2023-05-13 09:38:37,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:37,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:37,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:37,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:37,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:37,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:37,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:37,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:37,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:37,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:37,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:37,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:37,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:37,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:37,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:37,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:37,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +25: [2023-05-13 09:38:37,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:37,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:37,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:37,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:37,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:37,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:37,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +25: [2023-05-13 09:38:37,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:37,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:37,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +18: [2023-05-13 09:38:37,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:37,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:37,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:37,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:37,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:37,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:37,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:37,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:37,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:37,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:37,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:37,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:37,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:37,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:37,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +14: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:37,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:37,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:37,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:37,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:37,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +23: [2023-05-13 09:38:37,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:37,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:37,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:37,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:37,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:37,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:37,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:37,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +10: [2023-05-13 09:38:37,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:37,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:37,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:37,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:37,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:37,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:37,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:37,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:37,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:37,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:37,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:37,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:37,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:37,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:37,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:37,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:37,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +21: [2023-05-13 09:38:37,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:37,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:37,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:37,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:37,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:37,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:37,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:37,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +23: [2023-05-13 09:38:37,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:37,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +10: [2023-05-13 09:38:37,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:37,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:37,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +25: [2023-05-13 09:38:37,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:37,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:37,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:37,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:37,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:37,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:37,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:37,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:37,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:37,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:37,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:37,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:37,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:37,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:37,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:37,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:37,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:37,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:37,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:37,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:37,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +12: [2023-05-13 09:38:37,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +12: [2023-05-13 09:38:37,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +12: [2023-05-13 09:38:37,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +23: [2023-05-13 09:38:37,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:37,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:37,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:37,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:37,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:37,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:37,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:37,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:37,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:37,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:37,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:37,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:37,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:37,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:37,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:37,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:37,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:37,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:37,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:37,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:37,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:37,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +25: [2023-05-13 09:38:37,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:37,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:37,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:37,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:37,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:37,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:37,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:37,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:37,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:37,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:37,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:37,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:37,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:37,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:37,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:37,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:37,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:37,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +29: [2023-05-13 09:38:37,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:37,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:37,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +11: [2023-05-13 09:38:37,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:37,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:37,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:37,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:37,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:37,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:37,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:37,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:37,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:37,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +18: [2023-05-13 09:38:37,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +11: [2023-05-13 09:38:37,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:37,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:37,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:37,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:37,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:37,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:37,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +21: [2023-05-13 09:38:37,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:37,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:37,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:37,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:37,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:37,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:37,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:37,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:37,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:37,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:37,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:37,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:37,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:37,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:37,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:37,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:37,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:37,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:37,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:37,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:37,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +12: [2023-05-13 09:38:37,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:37,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:37,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:37,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:37,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:37,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:37,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:37,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:37,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +12: [2023-05-13 09:38:37,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:37,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:37,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:37,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:37,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:37,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:37,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:37,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:37,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:37,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:37,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:37,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:37,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:37,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:37,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:37,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:37,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:37,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:37,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:37,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:37,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:37,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +11: [2023-05-13 09:38:37,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:37,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +18: [2023-05-13 09:38:37,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:37,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:37,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +21: [2023-05-13 09:38:37,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:37,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:37,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:37,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:37,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:37,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +21: [2023-05-13 09:38:37,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:37,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:37,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:37,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +29: [2023-05-13 09:38:37,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:37,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:37,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:37,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:37,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:37,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:37,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:37,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:37,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:37,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:37,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:37,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:37,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:37,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:37,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:37,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:37,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:37,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:37,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:37,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:37,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:37,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:37,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:37,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:37,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:37,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:37,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:37,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +31: [2023-05-13 09:38:37,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:37,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:37,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:37,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:37,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:37,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:37,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:37,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:37,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:37,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:37,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:37,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:37,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:37,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:37,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:37,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:37,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:37,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +22: [2023-05-13 09:38:37,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:37,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:37,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:37,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:37,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:37,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:37,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:37,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:37,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:37,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:37,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:37,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:37,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:37,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:37,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:37,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:37,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:37,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +26: [2023-05-13 09:38:37,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +26: [2023-05-13 09:38:37,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +26: [2023-05-13 09:38:37,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:37,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:37,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:37,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:37,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:37,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:37,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +19: [2023-05-13 09:38:37,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:37,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:37,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:37,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +22: [2023-05-13 09:38:37,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:37,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:37,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +25: [2023-05-13 09:38:37,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:37,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:37,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:37,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:37,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:37,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +16: [2023-05-13 09:38:37,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:37,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:37,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:37,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:37,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +19: [2023-05-13 09:38:37,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:37,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:37,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:37,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:37,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:37,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:37,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:37,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:37,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:37,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:37,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:37,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:37,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:37,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:37,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:37,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:37,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:37,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:37,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:37,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:37,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:37,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:37,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:37,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:37,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:37,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:37,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:37,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:37,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:37,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:37,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:37,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:37,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:37,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:37,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:37,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:37,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:37,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:37,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:37,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:37,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:37,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:37,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:37,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:37,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:37,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:37,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:37,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:37,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:37,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +27: [2023-05-13 09:38:37,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +23: [2023-05-13 09:38:37,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:37,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:37,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:37,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:37,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:37,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:37,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:37,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:37,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:37,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:37,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:37,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:37,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:37,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:37,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:37,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:37,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:37,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:37,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:37,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:37,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:37,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:37,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +28: [2023-05-13 09:38:37,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:37,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:37,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:37,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:37,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:37,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:37,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:37,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:37,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:37,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:37,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:37,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:37,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:37,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:37,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:37,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:37,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:37,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 2: [2023-05-13 09:38:37,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:37,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:37,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +18: [2023-05-13 09:38:37,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +18: [2023-05-13 09:38:37,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +18: [2023-05-13 09:38:37,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:37,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:37,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +26: [2023-05-13 09:38:37,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:37,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:37,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:37,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:37,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:37,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:37,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:37,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:37,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:37,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:37,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:37,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:37,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:37,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:37,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:37,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:37,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:37,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:37,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:37,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:37,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:37,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:37,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:37,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:37,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:37,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:37,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:37,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:37,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:37,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:37,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:37,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:37,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:37,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:37,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:37,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:37,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:37,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:37,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:37,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:37,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:37,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:37,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:37,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:37,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:37,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:37,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:37,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:37,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:37,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:37,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:37,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:37,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:37,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +21: [2023-05-13 09:38:37,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +21: [2023-05-13 09:38:37,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +17: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +21: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:37,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:37,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +18: [2023-05-13 09:38:37,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:37,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:37,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:37,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:37,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:37,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:37,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:37,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:37,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +18: [2023-05-13 09:38:37,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:37,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:37,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:37,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:37,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +14: [2023-05-13 09:38:37,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +14: [2023-05-13 09:38:37,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +26: [2023-05-13 09:38:37,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:37,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:37,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:37,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +17: [2023-05-13 09:38:37,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:37,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:37,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:37,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:37,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +15: [2023-05-13 09:38:37,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +15: [2023-05-13 09:38:37,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:37,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:37,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:37,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:37,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:37,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:37,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +21: [2023-05-13 09:38:37,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:37,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +15: [2023-05-13 09:38:37,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +15: [2023-05-13 09:38:37,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:37,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:37,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:37,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:37,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:37,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:37,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:37,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:37,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:37,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:37,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:37,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:37,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:37,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:37,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:37,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 7: [2023-05-13 09:38:37,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:37,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:37,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:37,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:37,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:37,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +30: [2023-05-13 09:38:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:37,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:37,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:37,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:37,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:37,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:37,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +15: [2023-05-13 09:38:37,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +26: [2023-05-13 09:38:37,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:37,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:37,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:37,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:37,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:37,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:37,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:37,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:37,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:37,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:37,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +13: [2023-05-13 09:38:37,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:37,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:37,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:37,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:37,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:37,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:37,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:37,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:37,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:37,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:37,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:37,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:37,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:37,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:37,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:37,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:37,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 1: [2023-05-13 09:38:37,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:37,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:37,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:37,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:37,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +14: [2023-05-13 09:38:37,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:37,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:37,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:37,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:37,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +26: [2023-05-13 09:38:37,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:37,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:37,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:37,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:37,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:37,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:37,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:37,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +30: [2023-05-13 09:38:37,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:37,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +14: [2023-05-13 09:38:37,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +28: [2023-05-13 09:38:37,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:37,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +30: [2023-05-13 09:38:37,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +14: [2023-05-13 09:38:37,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:37,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:37,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +28: [2023-05-13 09:38:37,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +14: [2023-05-13 09:38:37,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +30: [2023-05-13 09:38:37,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:37,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:37,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +30: [2023-05-13 09:38:37,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +21: [2023-05-13 09:38:37,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:37,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:37,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:37,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:37,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +21: [2023-05-13 09:38:37,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:37,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +28: [2023-05-13 09:38:37,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +28: [2023-05-13 09:38:37,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:37,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +30: [2023-05-13 09:38:37,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:37,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:37,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:37,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:37,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:37,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:37,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +16: [2023-05-13 09:38:37,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:37,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:37,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:37,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:37,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:37,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:37,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:37,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:37,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:37,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:37,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +10: [2023-05-13 09:38:37,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +10: [2023-05-13 09:38:37,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +26: [2023-05-13 09:38:37,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:37,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +10: [2023-05-13 09:38:37,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +19: [2023-05-13 09:38:37,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:37,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +13: [2023-05-13 09:38:37,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:37,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:37,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:37,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:37,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:37,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:37,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:37,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:37,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:37,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:37,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:37,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:37,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:37,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:37,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +26: [2023-05-13 09:38:37,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +26: [2023-05-13 09:38:37,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:37,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:37,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:37,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:37,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:37,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:37,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:37,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +23: [2023-05-13 09:38:37,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +23: [2023-05-13 09:38:37,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +24: [2023-05-13 09:38:37,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +23: [2023-05-13 09:38:37,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:37,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:37,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:37,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:37,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:37,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:37,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:37,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:37,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:37,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:37,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:37,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +11: [2023-05-13 09:38:37,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +11: [2023-05-13 09:38:37,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +11: [2023-05-13 09:38:37,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:37,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:37,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:37,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:37,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:37,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:37,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:37,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:37,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:37,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:37,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:37,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:37,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:37,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:37,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:37,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +19: [2023-05-13 09:38:37,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +24: [2023-05-13 09:38:37,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:37,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +19: [2023-05-13 09:38:37,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +24: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:37,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:37,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +29: [2023-05-13 09:38:37,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +29: [2023-05-13 09:38:37,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +29: [2023-05-13 09:38:37,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:37,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:37,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:37,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +19: [2023-05-13 09:38:37,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:37,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:37,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:37,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:37,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:37,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:37,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:37,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:37,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +27: [2023-05-13 09:38:37,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +27: [2023-05-13 09:38:37,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +27: [2023-05-13 09:38:37,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +23: [2023-05-13 09:38:37,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:37,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +31: [2023-05-13 09:38:37,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:37,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:37,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:37,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:37,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:37,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:37,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:37,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:37,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:37,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:37,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:37,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +11: [2023-05-13 09:38:37,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:37,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +11: [2023-05-13 09:38:37,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +29: [2023-05-13 09:38:37,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:37,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:37,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:37,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:37,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:37,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:37,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:37,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:37,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:37,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:37,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:37,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:37,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:37,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:37,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:37,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:37,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +24: [2023-05-13 09:38:37,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:37,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:37,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:37,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:37,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:37,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:37,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:37,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +17: [2023-05-13 09:38:37,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +17: [2023-05-13 09:38:37,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +17: [2023-05-13 09:38:37,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:37,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:37,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:37,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:37,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:37,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:37,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:37,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:37,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:37,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:37,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:37,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:37,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:37,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:37,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:37,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:37,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:37,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:37,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:37,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:37,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:37,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:37,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:37,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:37,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +29: [2023-05-13 09:38:37,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +29: [2023-05-13 09:38:37,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:37,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:37,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:37,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:37,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:37,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:37,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:37,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:37,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:37,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:37,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:37,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:37,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:37,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:37,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:37,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:37,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:37,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:37,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:37,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:37,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:37,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:37,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:37,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +30: [2023-05-13 09:38:37,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +30: [2023-05-13 09:38:37,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +30: [2023-05-13 09:38:37,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:37,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 5: [2023-05-13 09:38:37,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:37,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +14: [2023-05-13 09:38:37,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:37,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:37,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:37,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:37,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:37,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:37,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:37,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:37,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:37,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:37,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:37,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:37,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:37,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:37,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:37,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:37,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:37,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:37,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:37,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:37,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:37,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:37,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:37,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:37,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:37,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:37,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:37,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:37,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:37,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:37,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:37,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:37,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:37,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:37,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:37,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:37,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:37,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:37,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:37,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:37,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 3: [2023-05-13 09:38:37,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:37,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:37,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:37,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:37,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 3: [2023-05-13 09:38:37,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:37,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:37,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:37,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:37,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:37,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:37,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:37,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:37,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:37,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +25: [2023-05-13 09:38:37,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:37,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:37,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:37,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:37,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:37,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:37,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:37,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:37,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:37,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:37,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:37,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:37,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 1: [2023-05-13 09:38:37,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:37,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:37,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:38,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +28: [2023-05-13 09:38:38,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +28: [2023-05-13 09:38:38,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +28: [2023-05-13 09:38:38,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:38,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:38,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:38,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:38,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:38,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:38,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:38,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:38,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:38,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:38,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:38,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:38,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:38,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:38,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:38,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +30: [2023-05-13 09:38:38,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +15: [2023-05-13 09:38:38,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:38,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:38,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:38,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:38,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:38,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:38,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:38,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +28: [2023-05-13 09:38:38,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:38,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:38,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:38,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:38,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:38,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:38,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +23: [2023-05-13 09:38:38,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +23: [2023-05-13 09:38:38,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +23: [2023-05-13 09:38:38,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:38,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:38,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +10: [2023-05-13 09:38:38,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +10: [2023-05-13 09:38:38,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +21: [2023-05-13 09:38:38,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +21: [2023-05-13 09:38:38,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +10: [2023-05-13 09:38:38,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:38,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:38,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:38,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:38,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:38,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +18: [2023-05-13 09:38:38,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:38,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:38,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +18: [2023-05-13 09:38:38,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:38,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:38,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:38,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:38,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:38,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:38,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:38,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +26: [2023-05-13 09:38:38,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +26: [2023-05-13 09:38:38,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +23: [2023-05-13 09:38:38,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:38,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:38,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:38,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:38,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:38,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:38,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:38,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:38,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:38,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:38,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:38,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:38,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:38,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +18: [2023-05-13 09:38:38,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +28: [2023-05-13 09:38:38,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +21: [2023-05-13 09:38:38,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +19: [2023-05-13 09:38:38,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:38,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +26: [2023-05-13 09:38:38,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +26: [2023-05-13 09:38:38,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:38,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:38,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:38,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:38,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:38,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +20: [2023-05-13 09:38:38,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +26: [2023-05-13 09:38:38,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:38,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +26: [2023-05-13 09:38:38,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:38,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:38,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:38,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:38,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:38,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:38,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:38,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:38,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:38,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:38,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:38,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:38,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +11: [2023-05-13 09:38:38,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:38,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:38,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:38,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:38,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:38,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +28: [2023-05-13 09:38:38,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:38,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:38,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:38,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:38,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:38,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:38,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:38,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:38,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:38,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:38,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:38,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:38,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:38,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:38,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:38,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:38,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:38,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +11: [2023-05-13 09:38:38,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:38,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:38,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:38,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:38,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:38,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:38,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:38,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:38,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:38,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +18: [2023-05-13 09:38:38,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:38,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:38,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:38,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:38,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:38,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:38,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:38,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:38,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:38,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:38,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +27: [2023-05-13 09:38:38,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +27: [2023-05-13 09:38:38,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +27: [2023-05-13 09:38:38,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +18: [2023-05-13 09:38:38,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:38,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:38,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +21: [2023-05-13 09:38:38,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +21: [2023-05-13 09:38:38,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:38,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:38,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:38,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:38,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:38,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:38,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +17: [2023-05-13 09:38:38,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:38,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:38,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:38,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +21: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:38,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:38,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +28: [2023-05-13 09:38:38,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:38,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:38,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +31: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +13: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:38,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:38,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +21: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:38,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:38,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:38,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:38,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:38,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:38,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:38,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:38,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:38,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:38,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:38,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +13: [2023-05-13 09:38:38,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:38,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:38,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:38,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:38,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:38,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:38,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:38,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:38,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:38,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:38,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:38,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:38,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:38,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:38,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:38,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:38,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:38,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:38,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:38,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:38,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:38,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:38,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:38,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:38,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +13: [2023-05-13 09:38:38,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:38,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:38,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:38,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:38,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:38,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +24: [2023-05-13 09:38:38,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:38,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:38,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +11: [2023-05-13 09:38:38,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +31: [2023-05-13 09:38:38,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:38,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:38,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:38,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:38,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:38,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:38,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:38,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +22: [2023-05-13 09:38:38,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:38,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:38,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:38,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:38,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:38,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:38,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +22: [2023-05-13 09:38:38,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:38,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:38,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:38,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:38,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:38,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:38,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +27: [2023-05-13 09:38:38,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +12: [2023-05-13 09:38:38,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:38,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:38,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:38,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +30: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +30: [2023-05-13 09:38:38,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +30: [2023-05-13 09:38:38,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +30: [2023-05-13 09:38:38,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:38,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:38,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:38,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:38,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:38,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +14: [2023-05-13 09:38:38,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +14: [2023-05-13 09:38:38,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +17: [2023-05-13 09:38:38,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +27: [2023-05-13 09:38:38,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:38,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:38,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:38,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:38,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:38,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +27: [2023-05-13 09:38:38,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +24: [2023-05-13 09:38:38,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:38,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:38,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:38,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:38,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +24: [2023-05-13 09:38:38,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:38,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +17: [2023-05-13 09:38:38,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:38,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:38,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:38,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:38,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +12: [2023-05-13 09:38:38,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:38,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:38,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:38,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +24: [2023-05-13 09:38:38,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:38,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:38,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:38,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:38,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:38,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:38,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:38,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +29: [2023-05-13 09:38:38,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +29: [2023-05-13 09:38:38,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +29: [2023-05-13 09:38:38,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +29: [2023-05-13 09:38:38,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +12: [2023-05-13 09:38:38,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:38,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:38,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:38,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +12: [2023-05-13 09:38:38,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +12: [2023-05-13 09:38:38,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +12: [2023-05-13 09:38:38,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:38,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +22: [2023-05-13 09:38:38,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:38,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:38,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +22: [2023-05-13 09:38:38,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:38,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:38,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:38,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... +22: [2023-05-13 09:38:38,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:38,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:38,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:38,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +15: [2023-05-13 09:38:38,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:38,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:38,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:38,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:38,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:38,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:38,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:38,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:38,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:38,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:38,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:38,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +28: [2023-05-13 09:38:38,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +28: [2023-05-13 09:38:38,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +28: [2023-05-13 09:38:38,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:38,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +15: [2023-05-13 09:38:38,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +15: [2023-05-13 09:38:38,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +15: [2023-05-13 09:38:38,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +15: [2023-05-13 09:38:38,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:38,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:38,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:38,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:38,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:38,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +11: [2023-05-13 09:38:38,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:38,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:38,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:38,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:38,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:38,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:38,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:38,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:38,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:38,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +10: [2023-05-13 09:38:38,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:38,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +11: [2023-05-13 09:38:38,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:38,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:38,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:38,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:38,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:38,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:38,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:38,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:38,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:38,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:38,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:38,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:38,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:38,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:38,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:38,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:38,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:38,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:38,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:38,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:38,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:38,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:38,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:38,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:38,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:38,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:38,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:38,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:38,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:38,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:38,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:38,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:38,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:38,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:38,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:38,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:38,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:38,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:38,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:38,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:38,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:38,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:38,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:38,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +24: [2023-05-13 09:38:38,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:38,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:38,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:38,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:38,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:38,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:38,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:38,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:38,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:38,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:38,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:38,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:38,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +25: [2023-05-13 09:38:38,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:38,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:38,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:38,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:38,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:38,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:38,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:38,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:38,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +25: [2023-05-13 09:38:38,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:38,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:38,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:38,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:38,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:38,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +23: [2023-05-13 09:38:38,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +23: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:38,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:38,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:38,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:38,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:38,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:38,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:38,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:38,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:38,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:38,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:38,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:38,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:38,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:38,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:38,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:38,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:38,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:38,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:38,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:38,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:38,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:38,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:38,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:38,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:38,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:38,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:38,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:38,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:38,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:38,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:38,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:38,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:38,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:38,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:38,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:38,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:38,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:38,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:38,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:38,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:38,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:38,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:38,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:38,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:38,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:38,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:38,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:38,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:38,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:38,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:38,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 1: [2023-05-13 09:38:38,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:38,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:38,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:38,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:38,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:38,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:38,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:38,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:38,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:38,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:38,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:38,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:38,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:38,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:38,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:38,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:38,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:38,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:38,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:38,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:38,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:38,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:38,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:38,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:38,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:38,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:38,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +21: [2023-05-13 09:38:38,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:38,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:38,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:38,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:38,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:38,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:38,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:38,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +27: [2023-05-13 09:38:38,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:38,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:38,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +31: [2023-05-13 09:38:38,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +27: [2023-05-13 09:38:38,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:38,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:38,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +21: [2023-05-13 09:38:38,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:38,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +31: [2023-05-13 09:38:38,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 1: [2023-05-13 09:38:38,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:38,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:38,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:38,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:38,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +13: [2023-05-13 09:38:38,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:38,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +12: [2023-05-13 09:38:38,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:38,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:38,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +12: [2023-05-13 09:38:38,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +23: [2023-05-13 09:38:38,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:38,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:38,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +21: [2023-05-13 09:38:38,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:38,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:38,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:38,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:38,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:38,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +26: [2023-05-13 09:38:38,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:38,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:38,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +26: [2023-05-13 09:38:38,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +31: [2023-05-13 09:38:38,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:38,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:38,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:38,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:38,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:38,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:38,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:38,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:38,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:38,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:38,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:38,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:38,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:38,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:38,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:38,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:38,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:38,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:38,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:38,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:38,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:38,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:38,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:38,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:38,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:38,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:38,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:38,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:38,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:38,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:38,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:38,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:38,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:38,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:38,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:38,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:38,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:38,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:38,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:38,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:38,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:38,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:38,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:38,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:38,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:38,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:38,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:38,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:38,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:38,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:38,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:38,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:38,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:38,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +14: [2023-05-13 09:38:38,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:38,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:38,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:38,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:38,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:38,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:38,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +24: [2023-05-13 09:38:38,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +17: [2023-05-13 09:38:38,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +17: [2023-05-13 09:38:38,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +17: [2023-05-13 09:38:38,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +31: [2023-05-13 09:38:38,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:38,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:38,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:38,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:38,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:38,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +14: [2023-05-13 09:38:38,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:38,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:38,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:38,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:38,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:38,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:38,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:38,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. +22: [2023-05-13 09:38:38,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 1: [2023-05-13 09:38:38,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:38,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:38,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +22: [2023-05-13 09:38:38,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +24: [2023-05-13 09:38:38,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:38,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:38,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:38,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:38,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:38,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:38,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:38,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:38,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:38,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:38,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:38,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:38,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:38,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:38,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:38,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:38,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:38,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:38,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:38,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:38,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:38,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:38,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:38,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:38,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:38,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +12: [2023-05-13 09:38:38,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:38,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:38,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:38,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:38,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:38,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:38,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:38,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:38,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +12: [2023-05-13 09:38:38,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:38,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:38,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:38,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:38,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:38,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:38,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:38,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:38,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:38,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:38,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:38,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:38,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:38,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:38,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:38,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:38,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:38,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:38,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:38,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:38,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:38,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:38,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:38,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:38,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:38,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:38,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:38,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:38,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:38,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:38,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:38,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:38,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:38,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:38,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:38,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +24: [2023-05-13 09:38:38,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:38,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:38,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +18: [2023-05-13 09:38:38,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +24: [2023-05-13 09:38:38,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +24: [2023-05-13 09:38:38,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:38,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:38,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:38,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:38,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +18: [2023-05-13 09:38:38,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:38,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:38,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:38,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:38,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:38,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:38,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:38,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +22: [2023-05-13 09:38:38,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:38,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:38,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +22: [2023-05-13 09:38:38,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt... +29: [2023-05-13 09:38:38,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +29: [2023-05-13 09:38:38,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +29: [2023-05-13 09:38:38,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:38,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:38,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:38,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:38,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:38,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +19: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:38,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:38,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:38,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:38,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:38,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:38,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:38,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +17: [2023-05-13 09:38:38,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:38,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:38,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +19: [2023-05-13 09:38:38,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:38,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:38,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:38,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:38,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:38,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:38,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:38,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:38,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:38,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:38,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:38,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:38,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +29: [2023-05-13 09:38:38,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:38,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:38,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:38,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:38,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:38,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:38,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:38,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:38,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:38,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:38,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:38,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:38,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:38,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:38,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:38,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:38,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:38,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:38,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:38,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:38,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:38,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:38,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:38,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:38,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:38,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:38,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:38,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:38,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:38,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:38,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:38,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:38,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:38,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:38,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:38,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +16: [2023-05-13 09:38:38,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:38,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:38,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +16: [2023-05-13 09:38:38,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:38,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:38,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:38,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:38,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:38,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:38,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:38,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:38,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:38,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:38,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:38,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:38,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:38,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:38,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:38,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:38,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:38,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:38,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:38,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:38,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:38,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:38,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:38,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:38,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +27: [2023-05-13 09:38:38,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +29: [2023-05-13 09:38:38,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:38,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:38,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +11: [2023-05-13 09:38:38,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:38,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:38,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +29: [2023-05-13 09:38:38,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +11: [2023-05-13 09:38:38,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:38,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:38,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:38,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:38,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:38,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:38,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:38,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:38,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:38,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:38,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:38,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:38,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:38,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:38,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:38,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:38,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:38,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 0: [2023-05-13 09:38:38,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:38,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:38,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:38,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:38,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:38,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:38,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:38,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:38,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +27: [2023-05-13 09:38:38,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:38,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:38,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:38,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:38,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:38,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +14: [2023-05-13 09:38:38,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +14: [2023-05-13 09:38:38,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +14: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:38,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:38,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:38,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:38,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:38,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:38,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:38,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:38,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:38,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:38,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +19: [2023-05-13 09:38:38,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:38,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:38,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:38,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:38,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:38,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:38,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:38,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:38,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_22-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:38,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:38,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:38,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:38,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:38,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +22: [2023-05-13 09:38:38,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:38,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:38,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +11: [2023-05-13 09:38:38,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +11: [2023-05-13 09:38:38,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +11: [2023-05-13 09:38:38,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:38,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:38,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:38,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:38,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:38,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:38,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:38,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:38,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:38,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:38,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:38,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:38,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:38,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:38,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:38,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:38,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:38,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:38,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:38,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:38,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:38,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:38,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:38,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:38,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:38,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:38,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:38,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:38,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:38,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:38,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +22: [2023-05-13 09:38:38,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:38,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:38,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:38,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:38,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:38,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:38,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:38,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:38,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:38,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:38,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:38,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:38,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:38,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:38,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:38,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:38,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:38,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:38,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:38,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:38,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:38,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:38,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:38,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:38,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:38,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:38,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:38,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:38,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:38,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:38,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:38,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:38,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:38,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:38,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +23: [2023-05-13 09:38:38,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:38,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:38,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:38,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:38,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +12: [2023-05-13 09:38:38,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:38,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:38,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:38,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:38,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:38,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:38,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +10: [2023-05-13 09:38:38,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:38,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +10: [2023-05-13 09:38:38,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:38,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:38,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:38,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:38,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:38,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:38,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:38,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:38,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:38,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:38,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:38,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:38,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:38,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:38,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:38,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:38,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:38,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:38,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +20: [2023-05-13 09:38:38,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:38,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:38,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:38,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:38,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +23: [2023-05-13 09:38:38,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:38,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +12: [2023-05-13 09:38:38,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:38,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:38,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:38,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:38,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:38,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:38,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:38,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:38,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +28: [2023-05-13 09:38:38,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +28: [2023-05-13 09:38:38,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +28: [2023-05-13 09:38:38,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:38,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:38,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:38,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:38,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:38,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:38,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:38,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:38,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:38,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:38,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:38,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:38,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:38,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +10: [2023-05-13 09:38:38,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:38,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:38,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:38,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:38,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:38,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:38,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:38,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:38,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:38,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:38,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:38,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:38,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:38,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:38,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:38,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:38,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:38,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:38,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:38,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:38,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:38,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:38,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:38,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:38,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:38,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:38,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:38,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:38,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:38,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:38,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:38,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:38,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:38,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:38,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:38,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:38,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:38,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:38,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +10: [2023-05-13 09:38:38,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +10: [2023-05-13 09:38:38,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +27: [2023-05-13 09:38:38,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:38,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:38,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:38,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:38,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:38,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:38,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:38,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:38,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:38,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +13: [2023-05-13 09:38:38,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 6: [2023-05-13 09:38:38,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +21: [2023-05-13 09:38:38,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:38,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:38,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:38,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 1: [2023-05-13 09:38:38,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:38,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:38,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +16: [2023-05-13 09:38:38,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:38,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:38,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:38,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:38,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:38,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:38,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:38,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +20: [2023-05-13 09:38:38,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:38,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:38,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:38,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:38,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:38,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:38,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:38,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:38,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:38,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:38,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:38,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:38,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:38,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:38,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:38,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:38,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:38,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:38,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:38,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:38,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:38,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:38,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:38,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:38,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:38,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:38,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +18: [2023-05-13 09:38:38,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:38,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:38,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:38,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:38,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:38,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:38,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:38,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:38,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:38,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +12: [2023-05-13 09:38:38,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +12: [2023-05-13 09:38:38,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +28: [2023-05-13 09:38:38,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:38,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:38,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:38,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:38,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:38,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:38,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:38,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:38,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:38,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:38,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:38,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:38,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:38,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:38,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:38,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 5: [2023-05-13 09:38:38,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:38,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:38,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:38,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:38,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:38,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:38,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:38,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:38,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:38,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:38,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:38,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:38,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:38,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:38,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +30: [2023-05-13 09:38:38,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:38,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:38,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:38,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:38,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:38,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:38,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:38,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:38,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:38,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:38,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:38,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:38,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:38,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:38,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:38,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +17: [2023-05-13 09:38:38,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:38,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:38,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:38,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:38,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:38,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:38,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:38,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:38,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:38,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:38,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +26: [2023-05-13 09:38:38,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:38,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:38,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +10: [2023-05-13 09:38:38,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:38,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +10: [2023-05-13 09:38:38,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:38,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +10: [2023-05-13 09:38:38,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:38,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +22: [2023-05-13 09:38:38,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:38,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:38,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:38,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:38,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:38,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:38,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +31: [2023-05-13 09:38:38,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:38,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:38,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +13: [2023-05-13 09:38:38,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:38,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:38,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:38,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:38,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:38,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:38,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:38,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +13: [2023-05-13 09:38:38,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:38,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:38,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:38,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:38,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:38,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:38,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +13: [2023-05-13 09:38:38,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:38,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:38,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:38,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:38,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:38,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:38,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:38,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:38,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:38,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:38,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:38,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:38,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:38,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:38,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:38,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:38,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:38,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:38,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +29: [2023-05-13 09:38:38,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +29: [2023-05-13 09:38:38,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:38,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:38,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:38,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:38,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:38,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:38,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +24: [2023-05-13 09:38:38,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +24: [2023-05-13 09:38:38,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +13: [2023-05-13 09:38:38,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:38,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:38,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:38,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:38,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:38,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +24: [2023-05-13 09:38:38,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:38,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:38,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:38,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:38,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:38,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:38,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:38,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:38,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:38,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:38,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:38,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:38,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:38,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:38,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:38,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:38,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:38,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:38,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:38,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:38,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:38,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:38,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:38,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:38,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:38,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:38,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:38,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +23: [2023-05-13 09:38:38,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:38,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:38,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:38,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:38,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:38,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:38,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:38,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:38,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +24: [2023-05-13 09:38:38,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:38,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:38,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:38,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:38,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:38,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:38,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:38,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:38,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:38,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:38,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:38,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:38,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +18: [2023-05-13 09:38:38,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:38,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:38,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:38,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:38,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:38,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:38,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:38,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:38,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:38,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:38,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:38,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:38,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:38,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:38,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:38,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +25: [2023-05-13 09:38:38,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:38,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:38,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:38,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:38,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:38,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +25: [2023-05-13 09:38:38,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:38,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:38,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:38,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:38,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:38,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:38,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:38,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:38,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:38,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:38,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:38,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:38,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:38,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:38,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:38,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:38,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:38,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:38,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:38,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:38,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:38,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +21: [2023-05-13 09:38:38,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:38,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:38,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:38,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:38,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:38,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:38,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:38,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:38,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:38,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:38,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:38,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:38,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:38,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:38,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:38,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:38,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:38,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:38,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +13: [2023-05-13 09:38:38,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:38,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:38,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:38,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:38,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:38,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:38,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:38,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:38,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:38,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:38,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:38,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:38,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:38,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:38,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:38,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:38,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:38,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:38,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:38,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:38,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:38,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:38,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:38,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:38,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:38,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:38,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:38,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:38,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:38,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:38,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:38,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:38,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:38,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:38,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:38,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:38,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:38,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:38,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:38,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:38,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:38,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:38,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:38,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +18: [2023-05-13 09:38:38,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:38,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:38,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:38,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:38,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:38,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:38,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:38,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:38,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:38,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:38,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:38,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:38,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:38,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:38,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +19: [2023-05-13 09:38:38,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:38,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:38,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:38,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:38,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:38,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:38,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:38,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:38,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:38,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:38,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:38,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:38,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:38,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:38,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:38,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:38,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:38,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:38,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:38,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:38,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:38,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:38,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:38,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:38,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:38,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:38,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:38,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:38,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:38,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:38,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:38,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:38,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:38,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:38,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:38,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:38,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:38,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:38,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:38,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:38,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:38,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:38,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:38,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:38,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:38,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 1: [2023-05-13 09:38:38,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:38,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:38,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:38,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:38,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:38,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:38,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:38,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +22: [2023-05-13 09:38:38,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 1: [2023-05-13 09:38:38,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. +22: [2023-05-13 09:38:38,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_23-model_01-model_states.pt. + 1: [2023-05-13 09:38:38,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:38,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:38,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:38,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:38,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:38,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:38,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:38,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:38,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:38,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:38,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:38,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:38,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:38,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:38,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:38,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:38,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:38,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:38,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +14: [2023-05-13 09:38:38,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 9: [2023-05-13 09:38:38,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:38,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +14: [2023-05-13 09:38:38,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:38,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:38,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:38,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:38,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:38,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:38,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:38,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:38,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:38,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:38,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:38,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:38,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:38,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +18: [2023-05-13 09:38:38,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:38,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:38,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:38,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 3: [2023-05-13 09:38:38,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:38,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:38,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:38,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:38,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:38,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:38,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:38,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:38,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:38,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:38,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:38,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:38,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:38,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:38,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:38,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:38,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:38,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:38,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:38,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:38,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:38,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:38,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:38,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:38,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:38,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:38,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:38,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:38,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:38,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 7: [2023-05-13 09:38:38,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +18: [2023-05-13 09:38:38,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:38,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:38,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:38,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:38,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:38,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:38,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:38,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:38,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:38,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:38,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:38,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:38,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:38,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:38,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:38,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:38,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:38,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:38,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:38,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:38,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:38,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:38,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:38,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:38,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:38,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:38,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:38,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +22: [2023-05-13 09:38:38,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... +21: [2023-05-13 09:38:38,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:38,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:38,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:38,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:38,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:38,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:38,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:38,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:38,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:38,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:38,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:38,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:38,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:38,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:38,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:38,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:38,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +11: [2023-05-13 09:38:38,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +14: [2023-05-13 09:38:38,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:38,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:38,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:38,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:38,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:38,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:38,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:38,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 1: [2023-05-13 09:38:38,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:38,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:38,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:38,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:38,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:38,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:38,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:38,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:38,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:38,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:38,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:38,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:38,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:38,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:38,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:38,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:38,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:38,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:38,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:38,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:38,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:38,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 3: [2023-05-13 09:38:38,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:38,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:38,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:38,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +24: [2023-05-13 09:38:38,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:38,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:38,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 9: [2023-05-13 09:38:38,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:38,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:38,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:38,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:38,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:38,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:38,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:38,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:38,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:38,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:38,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:38,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:38,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:38,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 7: [2023-05-13 09:38:38,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:38,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:38,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:38,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:38,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:38,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:38,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:38,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 2: [2023-05-13 09:38:38,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:38,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:38,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:38,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:38,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:38,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:38,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:38,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +27: [2023-05-13 09:38:38,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:38,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:38,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:38,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:38,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:38,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:38,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:38,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:38,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 4: [2023-05-13 09:38:38,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:38,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 8: [2023-05-13 09:38:38,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:38,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:38,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:38,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:38,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:38,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:38,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:38,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:38,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:38,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 5: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:38,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:38,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:38,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:38,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:38,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:38,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:38,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:38,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:38,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:38,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:38,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:39,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:39,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:39,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:39,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:39,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:39,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:39,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:39,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:39,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:39,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:39,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:39,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:39,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:39,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:39,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:39,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:39,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:39,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:39,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:39,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:39,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:39,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:39,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:39,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:39,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:39,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:39,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:39,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:39,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:39,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:39,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:39,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:39,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:39,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:39,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:39,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:39,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:39,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:39,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +13: [2023-05-13 09:38:39,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:39,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:39,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +27: [2023-05-13 09:38:39,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:39,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:39,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:39,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:39,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +27: [2023-05-13 09:38:39,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +13: [2023-05-13 09:38:39,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +16: [2023-05-13 09:38:39,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:39,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +13: [2023-05-13 09:38:39,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +16: [2023-05-13 09:38:39,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:39,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:39,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:39,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:39,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:39,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:39,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:39,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:39,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:39,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:39,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:39,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:39,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:39,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:39,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:39,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:39,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:39,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +19: [2023-05-13 09:38:39,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:39,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:39,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:39,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:39,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:39,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:39,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:39,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:39,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:39,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:39,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:39,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:39,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:39,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:39,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:39,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:39,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:39,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:39,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:39,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:39,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:39,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:39,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:39,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:39,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:39,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:39,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:39,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +20: [2023-05-13 09:38:39,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +19: [2023-05-13 09:38:39,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:39,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:39,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:39,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:39,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:39,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:39,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:39,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:39,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:39,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:39,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:39,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:39,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:39,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:39,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:39,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:39,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:39,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:39,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:39,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:39,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:39,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:39,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:39,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:39,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:39,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:39,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:39,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:39,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:39,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:39,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:39,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:39,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:39,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:39,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:39,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:39,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:39,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:39,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:39,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +13: [2023-05-13 09:38:39,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:39,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:39,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:39,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:39,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:39,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:39,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:39,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:39,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:39,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:39,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:39,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:39,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:39,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:39,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +27: [2023-05-13 09:38:39,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:39,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:39,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:39,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +27: [2023-05-13 09:38:39,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:39,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +24: [2023-05-13 09:38:39,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:39,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:39,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:39,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +16: [2023-05-13 09:38:39,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:39,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:39,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:39,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:39,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:39,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:39,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:39,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:39,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:39,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +13: [2023-05-13 09:38:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +13: [2023-05-13 09:38:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:39,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:39,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:39,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:39,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:39,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:39,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +27: [2023-05-13 09:38:39,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:39,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:39,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +10: [2023-05-13 09:38:39,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +10: [2023-05-13 09:38:39,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:39,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:39,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:39,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:39,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:39,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +19: [2023-05-13 09:38:39,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:39,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:39,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:39,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:39,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:39,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +24: [2023-05-13 09:38:39,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:39,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:39,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:39,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:39,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:39,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:39,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:39,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:39,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:39,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:39,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:39,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:39,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:39,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:39,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:39,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:39,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:39,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:39,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:39,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:39,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:39,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +26: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:39,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:39,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:39,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:39,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +22: [2023-05-13 09:38:39,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:39,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:39,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:39,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:39,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:39,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:39,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:39,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:39,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +21: [2023-05-13 09:38:39,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +26: [2023-05-13 09:38:39,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:39,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:39,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:39,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:39,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:39,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:39,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:39,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:39,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:39,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:39,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:39,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:39,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:39,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:39,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:39,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:39,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:39,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:39,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:39,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:39,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:39,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:39,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:39,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:39,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:39,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:39,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:39,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:39,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:39,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:39,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:39,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:39,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:39,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:39,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:39,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:39,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:39,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:39,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:39,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:39,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +14: [2023-05-13 09:38:39,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:39,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:39,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:39,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:39,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:39,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:39,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:39,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:39,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:39,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:39,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:39,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:39,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:39,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:39,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:39,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:39,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:39,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:39,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:39,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:39,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:39,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:39,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:39,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:39,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:39,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:39,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:39,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +10: [2023-05-13 09:38:39,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:39,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:39,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:39,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:39,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:39,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:39,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:39,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:39,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:39,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +26: [2023-05-13 09:38:39,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +26: [2023-05-13 09:38:39,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:39,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:39,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:39,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +14: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:39,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:39,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:39,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +26: [2023-05-13 09:38:39,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:39,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:39,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:39,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:39,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:39,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:39,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:39,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:39,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:39,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:39,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:39,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:39,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:39,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:39,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:39,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:39,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:39,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +14: [2023-05-13 09:38:39,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:39,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:39,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:39,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:39,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +17: [2023-05-13 09:38:39,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +17: [2023-05-13 09:38:39,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +17: [2023-05-13 09:38:39,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +15: [2023-05-13 09:38:39,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +15: [2023-05-13 09:38:39,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:39,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:39,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:39,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:39,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:39,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:39,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:39,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:39,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:39,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:39,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:39,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:39,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:39,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:39,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +22: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:39,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:39,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:39,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:39,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:39,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:39,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:39,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:39,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:39,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:39,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:39,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:39,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:39,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:39,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:39,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:39,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:39,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:39,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:39,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:39,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:39,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:39,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:39,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:39,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:39,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:39,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:39,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:39,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:39,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:39,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:39,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:39,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:39,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:39,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:39,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:39,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:39,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:39,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +14: [2023-05-13 09:38:39,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:39,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:39,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:39,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:39,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:39,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:39,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:39,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:39,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +11: [2023-05-13 09:38:39,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:39,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:39,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +26: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:39,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:39,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +26: [2023-05-13 09:38:39,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:39,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:39,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:39,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:39,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:39,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +26: [2023-05-13 09:38:39,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:39,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:39,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:39,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:39,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:39,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:39,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +26: [2023-05-13 09:38:39,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:39,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:39,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:39,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:39,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:39,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:39,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:39,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:39,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:39,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:39,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:39,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:39,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:39,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:39,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:39,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:39,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:39,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:39,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:39,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:39,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:39,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:39,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:39,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:39,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:39,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:39,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:39,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:39,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:39,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:39,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:39,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:39,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:39,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:39,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:39,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:39,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:39,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:39,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:39,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:39,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +17: [2023-05-13 09:38:39,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:39,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:39,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:39,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:39,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:39,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:39,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:39,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:39,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:39,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:39,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:39,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:39,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:39,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:39,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:39,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt... + 9: [2023-05-13 09:38:39,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:39,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +29: [2023-05-13 09:38:39,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:39,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 9: [2023-05-13 09:38:39,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:39,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:39,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:39,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:39,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:39,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:39,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:39,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:39,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +12: [2023-05-13 09:38:39,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:39,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:39,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:39,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:39,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:39,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +12: [2023-05-13 09:38:39,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:39,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:39,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:39,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:39,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:39,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:39,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:39,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:39,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:39,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:39,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:39,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:39,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:39,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:39,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +20: [2023-05-13 09:38:39,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:39,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:39,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:39,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:39,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:39,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:39,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:39,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:39,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:39,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:39,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:39,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:39,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:39,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:39,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:39,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:39,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:39,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:39,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:39,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:39,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:39,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:39,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:39,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:39,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:39,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:39,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:39,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:39,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:39,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:39,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:39,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:39,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:39,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:39,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +28: [2023-05-13 09:38:39,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:39,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:39,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:39,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:39,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:39,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:39,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:39,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:39,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:39,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:39,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +27: [2023-05-13 09:38:39,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +27: [2023-05-13 09:38:39,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +27: [2023-05-13 09:38:39,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:39,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:39,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +22: [2023-05-13 09:38:39,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:39,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:39,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:39,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:39,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:39,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:39,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:39,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:39,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:39,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:39,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:39,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:39,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:39,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:39,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:39,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:39,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:39,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:39,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:39,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:39,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:39,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:39,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:39,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:39,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:39,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:39,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:39,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:39,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:39,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:39,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:39,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:39,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:39,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:39,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:39,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:39,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:39,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:39,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:39,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:39,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:39,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:39,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:39,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:39,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:39,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:39,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:39,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:39,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:39,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:39,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:39,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:39,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:39,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:39,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:39,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:39,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:39,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:39,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +10: [2023-05-13 09:38:39,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:39,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:39,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:39,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:39,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:39,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +13: [2023-05-13 09:38:39,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:39,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:39,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:39,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:39,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:39,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +19: [2023-05-13 09:38:39,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:39,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +19: [2023-05-13 09:38:39,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:39,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:39,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:39,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:39,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:39,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:39,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:39,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +19: [2023-05-13 09:38:39,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +19: [2023-05-13 09:38:39,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:39,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +19: [2023-05-13 09:38:39,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:39,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:39,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:39,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:39,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:39,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:39,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:39,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:39,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +27: [2023-05-13 09:38:39,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:39,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:39,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:39,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:39,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:39,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:39,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +14: [2023-05-13 09:38:39,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:39,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:39,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:39,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:39,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:39,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +20: [2023-05-13 09:38:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:39,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:39,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:39,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:39,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:39,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:39,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:39,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:39,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:39,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:39,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:39,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:39,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:39,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:39,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:39,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:39,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:39,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:39,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:39,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:39,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:39,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:39,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:39,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:39,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:39,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:39,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:39,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:39,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:39,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:39,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:39,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:39,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:39,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +28: [2023-05-13 09:38:39,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:39,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:39,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:39,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +28: [2023-05-13 09:38:39,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:39,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:39,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:39,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:39,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:39,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:39,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:39,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:39,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:39,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +29: [2023-05-13 09:38:39,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +24: [2023-05-13 09:38:39,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:39,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:39,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:39,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:39,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:39,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:39,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:39,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:39,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:39,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:39,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:39,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:39,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:39,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:39,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:39,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:39,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:39,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:39,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:39,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:39,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:39,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:39,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:39,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:39,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +16: [2023-05-13 09:38:39,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:39,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +16: [2023-05-13 09:38:39,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:39,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:39,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:39,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:39,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:39,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:39,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:39,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:39,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:39,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:39,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:39,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:39,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:39,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:39,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +12: [2023-05-13 09:38:39,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:39,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:39,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +26: [2023-05-13 09:38:39,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:39,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:39,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:39,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:39,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:39,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:39,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:39,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:39,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:39,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:39,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:39,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:39,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:39,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:39,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:39,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:39,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:39,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +31: [2023-05-13 09:38:39,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:39,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +31: [2023-05-13 09:38:39,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:39,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +13: [2023-05-13 09:38:39,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:39,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +20: [2023-05-13 09:38:39,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:39,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:39,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +20: [2023-05-13 09:38:39,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +20: [2023-05-13 09:38:39,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +24: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +24: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +25: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +25: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +10: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +10: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +20: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +24: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +10: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:39,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:39,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:39,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:39,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:39,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:39,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:39,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:39,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:39,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:39,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:39,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:39,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:39,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:39,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:39,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:39,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:39,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:39,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:39,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +15: [2023-05-13 09:38:39,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +13: [2023-05-13 09:38:39,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:39,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:39,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:39,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:39,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:39,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:39,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:39,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +21: [2023-05-13 09:38:39,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:39,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:39,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:39,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:39,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +17: [2023-05-13 09:38:39,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +17: [2023-05-13 09:38:39,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +17: [2023-05-13 09:38:39,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:39,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:39,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:39,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:39,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:39,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:39,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:39,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:39,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:39,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:39,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:39,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:39,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:39,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:39,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:39,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:39,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:39,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:39,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:39,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:39,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:39,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:39,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:39,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:39,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:39,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:39,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:39,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:39,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:39,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:39,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:39,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:39,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:39,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:39,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:39,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:39,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:39,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:39,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:39,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:39,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:39,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:39,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:39,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:39,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:39,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:39,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:39,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:39,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:39,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:39,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:39,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:39,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:39,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:39,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +13: [2023-05-13 09:38:39,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:39,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:39,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:39,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:39,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +13: [2023-05-13 09:38:39,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +26: [2023-05-13 09:38:39,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:39,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:39,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:39,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:39,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:39,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:39,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +31: [2023-05-13 09:38:39,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:39,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:39,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:39,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:39,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:39,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +17: [2023-05-13 09:38:39,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:39,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:39,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:39,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +21: [2023-05-13 09:38:39,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +21: [2023-05-13 09:38:39,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +13: [2023-05-13 09:38:39,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:39,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:39,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:39,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:39,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:39,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:39,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:39,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:39,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +29: [2023-05-13 09:38:39,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +30: [2023-05-13 09:38:39,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:39,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +26: [2023-05-13 09:38:39,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +26: [2023-05-13 09:38:39,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:39,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:39,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:39,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:39,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:39,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +17: [2023-05-13 09:38:39,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:39,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:39,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +29: [2023-05-13 09:38:39,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +29: [2023-05-13 09:38:39,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +22: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +15: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +17: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:39,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +22: [2023-05-13 09:38:39,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:39,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +22: [2023-05-13 09:38:39,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:39,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:39,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:39,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:39,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:39,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:39,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +23: [2023-05-13 09:38:39,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:39,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:39,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:39,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:39,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:39,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:39,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +15: [2023-05-13 09:38:39,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:39,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:39,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:39,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:39,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:39,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +15: [2023-05-13 09:38:39,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:39,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:39,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:39,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:39,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:39,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:39,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:39,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:39,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:39,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:39,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:39,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:39,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:39,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:39,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:39,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:39,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:39,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +17: [2023-05-13 09:38:39,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +22: [2023-05-13 09:38:39,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:39,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:39,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +26: [2023-05-13 09:38:39,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:39,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +22: [2023-05-13 09:38:39,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:39,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:39,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:39,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:39,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:39,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:39,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:39,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:39,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:39,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:39,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:39,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:39,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:39,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:39,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:39,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:39,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:39,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:39,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:39,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:39,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:39,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +23: [2023-05-13 09:38:39,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... +22: [2023-05-13 09:38:39,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:39,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:39,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:39,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:39,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:39,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:39,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:39,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:39,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:39,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +12: [2023-05-13 09:38:39,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:39,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:39,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:39,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:39,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:39,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:39,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:39,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:39,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:39,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:39,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:39,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:39,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:39,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:39,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:39,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:39,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:39,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:39,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:39,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:39,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +16: [2023-05-13 09:38:39,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:39,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:39,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +10: [2023-05-13 09:38:39,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:39,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:39,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 9: [2023-05-13 09:38:39,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:39,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +18: [2023-05-13 09:38:39,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:39,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:39,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:39,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:39,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:39,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:39,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:39,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:39,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 4: [2023-05-13 09:38:39,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +19: [2023-05-13 09:38:39,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +22: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +23: [2023-05-13 09:38:39,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +23: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +22: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +16: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +16: [2023-05-13 09:38:39,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:39,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:39,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +11: [2023-05-13 09:38:39,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +11: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +11: [2023-05-13 09:38:39,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:39,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:39,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:39,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +18: [2023-05-13 09:38:39,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:39,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:39,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:39,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:39,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:39,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:39,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:39,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:39,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:39,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:39,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:39,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:39,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:39,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:39,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:39,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +16: [2023-05-13 09:38:39,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:39,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:39,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:39,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:39,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +16: [2023-05-13 09:38:39,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:39,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:39,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:39,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:39,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 3: [2023-05-13 09:38:39,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:39,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:39,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:39,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:39,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:39,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:39,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:39,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:39,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:39,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:39,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:39,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:39,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:39,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:39,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:39,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:39,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:39,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:39,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:39,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:39,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:39,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:39,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:39,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:39,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +19: [2023-05-13 09:38:39,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:39,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 4: [2023-05-13 09:38:39,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:39,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:39,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 9: [2023-05-13 09:38:39,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:39,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:39,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:39,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:39,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:39,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:39,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:39,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:39,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:39,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:39,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:39,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:39,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:39,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:39,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:39,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:39,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:39,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:39,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:39,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:39,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:39,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:39,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:39,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:39,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:39,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:39,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:39,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:39,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:39,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:39,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:39,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:39,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:39,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +12: [2023-05-13 09:38:39,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +12: [2023-05-13 09:38:39,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +12: [2023-05-13 09:38:39,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:39,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:39,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:39,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:39,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:39,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:39,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:39,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:39,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:39,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:39,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:39,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:39,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:39,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:39,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:39,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:39,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:39,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:39,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:39,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:39,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:39,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:39,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:39,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +30: [2023-05-13 09:38:39,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +30: [2023-05-13 09:38:39,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:39,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +21: [2023-05-13 09:38:39,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:39,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:39,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:39,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:39,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:39,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:39,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:39,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:39,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:39,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:39,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:39,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:39,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:39,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:39,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:39,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:39,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:39,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +28: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +28: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 5: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:39,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:39,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:39,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:39,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:39,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:39,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +12: [2023-05-13 09:38:39,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:39,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:39,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:39,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:39,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:39,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:39,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:39,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:39,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +30: [2023-05-13 09:38:39,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:39,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:39,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:39,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:39,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 7: [2023-05-13 09:38:39,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:39,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:39,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:39,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:39,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +20: [2023-05-13 09:38:39,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:39,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:39,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +31: [2023-05-13 09:38:39,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:39,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +28: [2023-05-13 09:38:39,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:39,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +31: [2023-05-13 09:38:39,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:39,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:39,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:39,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +13: [2023-05-13 09:38:39,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:39,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:39,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:39,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:39,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:39,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +23: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +23: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +28: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:39,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +28: [2023-05-13 09:38:39,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:39,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:39,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +28: [2023-05-13 09:38:39,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:39,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:39,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 5: [2023-05-13 09:38:39,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:39,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:39,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:39,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:39,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:39,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:39,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:39,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:39,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:39,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:39,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:39,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:39,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:39,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:39,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +21: [2023-05-13 09:38:39,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:39,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:39,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +15: [2023-05-13 09:38:39,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:39,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +31: [2023-05-13 09:38:39,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:39,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. + 0: [2023-05-13 09:38:39,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:39,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:39,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:39,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:39,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:39,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:39,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:39,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:39,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:39,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:39,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:39,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:39,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:39,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:39,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:39,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +25: [2023-05-13 09:38:39,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +25: [2023-05-13 09:38:39,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:39,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 0: [2023-05-13 09:38:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:39,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +20: [2023-05-13 09:38:39,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:39,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:39,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:39,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +20: [2023-05-13 09:38:39,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:39,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +13: [2023-05-13 09:38:39,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +22: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:39,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +15: [2023-05-13 09:38:39,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:39,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:39,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:39,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:39,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:39,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:39,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:39,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +13: [2023-05-13 09:38:39,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +22: [2023-05-13 09:38:39,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:39,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +22: [2023-05-13 09:38:39,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:39,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:39,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:39,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:39,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:39,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +10: [2023-05-13 09:38:39,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:39,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:39,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:39,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:39,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:39,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:39,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:39,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:39,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:39,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:39,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:39,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:39,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:39,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +15: [2023-05-13 09:38:39,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:39,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:39,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:39,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:39,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +25: [2023-05-13 09:38:39,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:39,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:39,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:39,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:39,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +17: [2023-05-13 09:38:39,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +17: [2023-05-13 09:38:39,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +21: [2023-05-13 09:38:39,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:39,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +25: [2023-05-13 09:38:39,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:39,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +10: [2023-05-13 09:38:39,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:39,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +15: [2023-05-13 09:38:39,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:39,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:39,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +10: [2023-05-13 09:38:39,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... + 8: [2023-05-13 09:38:39,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +25: [2023-05-13 09:38:39,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:39,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:39,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:39,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:39,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:39,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:39,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:39,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:39,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:39,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:39,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +25: [2023-05-13 09:38:39,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:39,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:39,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +22: [2023-05-13 09:38:39,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +22: [2023-05-13 09:38:39,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +10: [2023-05-13 09:38:39,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +22: [2023-05-13 09:38:39,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +20: [2023-05-13 09:38:39,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:39,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:39,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:39,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:39,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:39,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:39,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:39,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:39,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +29: [2023-05-13 09:38:39,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +29: [2023-05-13 09:38:39,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +29: [2023-05-13 09:38:39,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_26-model_01-model_states.pt. +17: [2023-05-13 09:38:39,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:39,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:39,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +26: [2023-05-13 09:38:39,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +26: [2023-05-13 09:38:39,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +26: [2023-05-13 09:38:39,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:39,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:39,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:39,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:39,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +29: [2023-05-13 09:38:39,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:40,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +17: [2023-05-13 09:38:40,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +17: [2023-05-13 09:38:40,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:40,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:40,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:40,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:40,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:40,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:40,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:40,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:40,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:40,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:40,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:40,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:40,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:40,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +29: [2023-05-13 09:38:40,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:40,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:40,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +29: [2023-05-13 09:38:40,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:40,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:40,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:40,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:40,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:40,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:40,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +24: [2023-05-13 09:38:40,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:40,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:40,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:40,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:40,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:40,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:40,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:40,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:40,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:40,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:40,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt... +13: [2023-05-13 09:38:40,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:40,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:40,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +11: [2023-05-13 09:38:40,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:40,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:40,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:40,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +19: [2023-05-13 09:38:40,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:40,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:40,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:40,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:40,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:40,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:40,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:40,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:40,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +13: [2023-05-13 09:38:40,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:40,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:40,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:40,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:40,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:40,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:40,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:40,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:40,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:40,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:40,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:40,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:40,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:40,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:40,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:40,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:40,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:40,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:40,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:40,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:40,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:40,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:40,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:40,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:40,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +14: [2023-05-13 09:38:40,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:40,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:40,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:40,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:40,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:40,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:40,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:40,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:40,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:40,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:40,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +14: [2023-05-13 09:38:40,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:40,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:40,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:40,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:40,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:40,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:40,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:40,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:40,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:40,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:40,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +18: [2023-05-13 09:38:40,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +18: [2023-05-13 09:38:40,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +18: [2023-05-13 09:38:40,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:40,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:40,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +24: [2023-05-13 09:38:40,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:40,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:40,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:40,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:40,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:40,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:40,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:40,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:40,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:40,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:40,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +21: [2023-05-13 09:38:40,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +19: [2023-05-13 09:38:40,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:40,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +21: [2023-05-13 09:38:40,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:40,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:40,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +23: [2023-05-13 09:38:40,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +14: [2023-05-13 09:38:40,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +23: [2023-05-13 09:38:40,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +30: [2023-05-13 09:38:40,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +30: [2023-05-13 09:38:40,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +30: [2023-05-13 09:38:40,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:40,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +24: [2023-05-13 09:38:40,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +14: [2023-05-13 09:38:40,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:40,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:40,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:40,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:40,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:40,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:40,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:40,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:40,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:40,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:40,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:40,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +11: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +11: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +11: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +15: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +30: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +21: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:40,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +25: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:40,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:40,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:40,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:40,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:40,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +27: [2023-05-13 09:38:40,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +23: [2023-05-13 09:38:40,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:40,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:40,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:40,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:40,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +24: [2023-05-13 09:38:40,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:40,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:40,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:40,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:40,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:40,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:40,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:40,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +11: [2023-05-13 09:38:40,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +27: [2023-05-13 09:38:40,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:40,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:40,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:40,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +11: [2023-05-13 09:38:40,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:40,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:40,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:40,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:40,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:40,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:40,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +21: [2023-05-13 09:38:40,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:40,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +21: [2023-05-13 09:38:40,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:40,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +21: [2023-05-13 09:38:40,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:40,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:40,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +12: [2023-05-13 09:38:40,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:40,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:40,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:40,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:40,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:40,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:40,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:40,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:40,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:40,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:40,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:40,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +12: [2023-05-13 09:38:40,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:40,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:40,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +30: [2023-05-13 09:38:40,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:40,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:40,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +12: [2023-05-13 09:38:40,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:40,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:40,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:40,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:40,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:40,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:40,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:40,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +12: [2023-05-13 09:38:40,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:40,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:40,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:40,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:40,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:40,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:40,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:40,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:40,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:40,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:40,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:40,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:40,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:40,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:40,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:40,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:40,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:40,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:40,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:40,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:40,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:40,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:40,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:40,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:40,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:40,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:40,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:40,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:40,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:40,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:40,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:40,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:40,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:40,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:40,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:40,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:40,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:40,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:40,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:40,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +11: [2023-05-13 09:38:40,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:40,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:40,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +12: [2023-05-13 09:38:40,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:40,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:40,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:40,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:40,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:40,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:40,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:40,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:40,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:40,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:40,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:40,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +12: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:40,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:40,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:40,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:40,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +15: [2023-05-13 09:38:40,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:40,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:40,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:40,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:40,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:40,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:40,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +20: [2023-05-13 09:38:40,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:40,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +11: [2023-05-13 09:38:40,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:40,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:40,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:40,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:40,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:40,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +12: [2023-05-13 09:38:40,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:40,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:40,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +13: [2023-05-13 09:38:40,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +13: [2023-05-13 09:38:40,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +12: [2023-05-13 09:38:40,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:40,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:40,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:40,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:40,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:40,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:40,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:40,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:40,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:40,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:40,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:40,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:40,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:40,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:40,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:40,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:40,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:40,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:40,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +30: [2023-05-13 09:38:40,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:40,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:40,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:40,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:40,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:40,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:40,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:40,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:40,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:40,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:40,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:40,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:40,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:40,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:40,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:40,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:40,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:40,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:40,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:40,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:40,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:40,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +20: [2023-05-13 09:38:40,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:40,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:40,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:40,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:40,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +15: [2023-05-13 09:38:40,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +14: [2023-05-13 09:38:40,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:40,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:40,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:40,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:40,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:40,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:40,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +15: [2023-05-13 09:38:40,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +14: [2023-05-13 09:38:40,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:40,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:40,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:40,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:40,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:40,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:40,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:40,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:40,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:40,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +17: [2023-05-13 09:38:40,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +17: [2023-05-13 09:38:40,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +26: [2023-05-13 09:38:40,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:40,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +26: [2023-05-13 09:38:40,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:40,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:40,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:40,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:40,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:40,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:40,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:40,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:40,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:40,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:40,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:40,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:40,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:40,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:40,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:40,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:40,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:40,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:40,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:40,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:40,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +13: [2023-05-13 09:38:40,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:40,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +30: [2023-05-13 09:38:40,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:40,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:40,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:40,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:40,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:40,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:40,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +27: [2023-05-13 09:38:40,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:40,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:40,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:40,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:40,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:40,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:40,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +13: [2023-05-13 09:38:40,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:40,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:40,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:40,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:40,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:40,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:40,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:40,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:40,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:40,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:40,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:40,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:40,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +30: [2023-05-13 09:38:40,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:40,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:40,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +27: [2023-05-13 09:38:40,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:40,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:40,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:40,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:40,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:40,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +30: [2023-05-13 09:38:40,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:40,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:40,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:40,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +23: [2023-05-13 09:38:40,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:40,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:40,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:40,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:40,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:40,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +27: [2023-05-13 09:38:40,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:40,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:40,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:40,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:40,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:40,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:40,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:40,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:40,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:40,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:40,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:40,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:40,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:40,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:40,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:40,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:40,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:40,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:40,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:40,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:40,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:40,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:40,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:40,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +29: [2023-05-13 09:38:40,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:40,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:40,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. +17: [2023-05-13 09:38:40,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +17: [2023-05-13 09:38:40,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +29: [2023-05-13 09:38:40,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:40,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:40,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:40,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:40,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:40,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:40,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +25: [2023-05-13 09:38:40,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:40,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:40,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +25: [2023-05-13 09:38:40,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:40,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:40,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:40,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:40,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:40,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:40,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:40,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:40,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:40,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:40,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:40,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:40,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:40,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:40,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:40,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:40,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:40,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:40,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:40,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:40,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +17: [2023-05-13 09:38:40,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:40,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:40,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +21: [2023-05-13 09:38:40,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:40,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:40,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:40,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:40,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:40,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:40,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:40,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +17: [2023-05-13 09:38:40,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:40,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:40,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:40,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:40,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:40,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:40,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +17: [2023-05-13 09:38:40,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:40,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:40,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +29: [2023-05-13 09:38:40,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt... +25: [2023-05-13 09:38:40,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:40,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:40,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:40,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:40,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:40,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:40,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:40,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:40,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:40,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:40,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:40,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:40,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:40,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:40,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:40,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:40,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:40,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +26: [2023-05-13 09:38:40,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:40,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:40,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:40,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +16: [2023-05-13 09:38:40,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:40,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:40,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:40,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:40,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:40,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:40,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:40,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:40,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:40,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:40,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:40,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:40,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:40,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:40,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:40,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +29: [2023-05-13 09:38:40,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:40,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:40,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:40,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:40,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:40,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:40,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:40,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:40,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +28: [2023-05-13 09:38:40,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:40,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +28: [2023-05-13 09:38:40,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:40,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:40,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:40,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:40,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:40,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:40,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:40,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:40,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:40,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:40,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:40,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:40,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:40,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:40,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:40,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:40,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:40,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:40,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:40,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:40,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:40,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:40,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:40,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:40,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:40,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:40,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:40,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:40,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:40,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:40,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:40,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:40,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:40,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:40,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:40,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:40,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:40,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:40,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:40,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:40,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:40,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:40,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:40,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:40,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:40,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:40,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:40,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:40,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:40,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:40,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:40,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:40,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:40,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:40,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:40,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:40,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:40,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:40,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:40,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:40,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:40,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:40,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:40,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:40,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:40,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:40,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:40,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:40,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:40,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:40,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:40,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:40,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:40,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:40,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:40,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:40,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +27: [2023-05-13 09:38:40,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:40,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:40,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:40,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:40,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:40,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:40,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:40,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:40,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:40,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:40,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:40,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:40,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:40,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:40,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:40,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:40,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:40,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:40,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:40,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:40,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:40,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:40,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:40,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:40,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:40,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:40,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:40,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:40,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:40,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:40,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:40,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:40,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:40,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:40,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:40,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:40,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:40,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:40,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:40,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:40,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:40,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:40,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:40,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:40,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:40,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:40,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:40,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:40,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:40,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:40,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:40,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:40,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:40,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:40,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:40,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:40,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:40,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:40,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:40,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:40,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:40,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:40,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:40,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:40,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:40,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:40,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:40,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:40,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:40,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:40,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:40,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:40,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:40,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:40,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +28: [2023-05-13 09:38:40,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +28: [2023-05-13 09:38:40,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +28: [2023-05-13 09:38:40,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +28: [2023-05-13 09:38:40,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:40,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:40,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:40,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:40,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:40,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:40,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:40,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:40,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:40,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:40,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:40,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:40,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:40,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:40,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:40,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:40,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:40,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:40,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:40,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:40,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:40,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:40,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:40,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:40,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:40,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:40,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:40,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:40,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:40,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:40,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:40,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:40,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:40,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:40,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:40,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:40,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:40,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:40,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:40,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:40,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:40,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:40,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:40,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:40,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:40,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:40,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:40,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:40,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:40,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:40,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:40,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:40,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:40,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:40,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:40,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +24: [2023-05-13 09:38:40,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +12: [2023-05-13 09:38:40,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:40,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:40,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:40,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:40,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:40,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:40,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:40,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:40,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:40,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +22: [2023-05-13 09:38:40,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:40,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +12: [2023-05-13 09:38:40,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:40,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:40,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:40,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:40,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:40,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +22: [2023-05-13 09:38:40,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:40,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:40,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:40,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:40,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:40,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:40,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:40,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:40,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:40,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +31: [2023-05-13 09:38:40,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:40,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:40,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +31: [2023-05-13 09:38:40,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:40,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:40,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:40,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:40,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:40,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:40,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:40,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:40,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:40,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:40,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:40,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:40,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:40,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:40,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:40,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:40,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:40,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:40,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:40,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +28: [2023-05-13 09:38:40,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +20: [2023-05-13 09:38:40,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:40,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:40,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:40,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:40,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:40,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:40,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +12: [2023-05-13 09:38:40,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:40,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:40,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:40,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:40,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:40,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:40,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:40,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:40,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:40,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:40,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:40,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:40,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +21: [2023-05-13 09:38:40,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:40,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:40,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:40,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:40,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:40,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:40,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:40,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:40,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:40,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:40,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:40,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:40,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:40,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:40,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:40,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:40,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:40,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +28: [2023-05-13 09:38:40,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:40,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:40,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:40,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:40,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:40,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:40,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:40,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:40,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:40,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:40,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:40,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:40,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +21: [2023-05-13 09:38:40,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:40,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:40,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:40,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:40,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:40,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:40,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:40,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:40,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +24: [2023-05-13 09:38:40,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:40,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +24: [2023-05-13 09:38:40,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:40,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:40,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:40,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:40,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:40,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:40,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +31: [2023-05-13 09:38:40,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:40,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +21: [2023-05-13 09:38:40,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +21: [2023-05-13 09:38:40,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +31: [2023-05-13 09:38:40,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +31: [2023-05-13 09:38:40,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:40,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:40,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +24: [2023-05-13 09:38:40,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:40,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:40,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:40,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +31: [2023-05-13 09:38:40,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:40,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +21: [2023-05-13 09:38:40,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:40,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:40,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:40,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:40,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:40,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:40,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:40,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:40,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:40,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +13: [2023-05-13 09:38:40,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +13: [2023-05-13 09:38:40,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +16: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +16: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 6: [2023-05-13 09:38:40,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:40,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:40,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +21: [2023-05-13 09:38:40,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:40,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +23: [2023-05-13 09:38:40,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +23: [2023-05-13 09:38:40,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +23: [2023-05-13 09:38:40,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:40,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:40,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:40,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:40,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:40,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +23: [2023-05-13 09:38:40,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:40,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:40,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:40,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:40,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +21: [2023-05-13 09:38:40,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:40,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:40,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:40,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:40,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +21: [2023-05-13 09:38:40,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +21: [2023-05-13 09:38:40,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:40,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:40,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:40,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:40,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:40,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:40,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:40,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:40,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +18: [2023-05-13 09:38:40,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:40,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:40,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:40,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:40,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:40,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:40,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:40,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:40,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:40,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:40,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:40,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:40,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:40,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:40,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:40,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:40,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:40,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:40,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:40,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:40,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:40,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:40,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:40,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:40,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:40,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:40,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:40,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:40,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:40,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +20: [2023-05-13 09:38:40,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:40,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:40,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:40,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +16: [2023-05-13 09:38:40,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:40,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:40,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +12: [2023-05-13 09:38:40,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:40,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +12: [2023-05-13 09:38:40,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +13: [2023-05-13 09:38:40,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:40,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +29: [2023-05-13 09:38:40,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:40,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +13: [2023-05-13 09:38:40,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +13: [2023-05-13 09:38:40,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:40,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +11: [2023-05-13 09:38:40,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:40,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:40,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:40,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:40,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +11: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +11: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +13: [2023-05-13 09:38:40,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +11: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +11: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +10: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:40,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +17: [2023-05-13 09:38:40,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:40,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:40,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +10: [2023-05-13 09:38:40,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:40,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:40,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:40,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +20: [2023-05-13 09:38:40,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +13: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +10: [2023-05-13 09:38:40,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +12: [2023-05-13 09:38:40,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:40,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:40,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:40,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:40,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:40,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 8: [2023-05-13 09:38:40,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +30: [2023-05-13 09:38:40,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:40,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:40,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:40,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:40,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:40,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:40,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:40,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:40,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:40,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:40,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:40,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:40,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:40,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +27: [2023-05-13 09:38:40,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +27: [2023-05-13 09:38:40,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:40,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:40,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +27: [2023-05-13 09:38:40,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:40,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:40,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:40,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:40,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:40,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:40,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +20: [2023-05-13 09:38:40,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +20: [2023-05-13 09:38:40,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +29: [2023-05-13 09:38:40,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +12: [2023-05-13 09:38:40,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:40,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +12: [2023-05-13 09:38:40,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +12: [2023-05-13 09:38:40,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +29: [2023-05-13 09:38:40,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +27: [2023-05-13 09:38:40,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:40,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +27: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:40,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 7: [2023-05-13 09:38:40,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 7: [2023-05-13 09:38:40,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:40,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:40,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:40,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:40,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:40,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:40,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:40,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:40,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:40,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:40,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:40,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:40,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +30: [2023-05-13 09:38:40,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:40,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +29: [2023-05-13 09:38:40,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:40,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:40,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:40,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:40,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:40,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:40,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:40,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:40,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:40,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:40,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:40,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:40,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +30: [2023-05-13 09:38:40,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:40,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:40,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:40,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:40,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:40,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:40,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:40,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:40,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:40,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:40,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:40,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:40,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:40,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:40,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:40,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:40,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:40,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:40,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +15: [2023-05-13 09:38:40,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +15: [2023-05-13 09:38:40,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +15: [2023-05-13 09:38:40,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:40,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:40,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:40,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +28: [2023-05-13 09:38:40,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +28: [2023-05-13 09:38:40,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +27: [2023-05-13 09:38:40,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +28: [2023-05-13 09:38:40,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:40,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:40,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:40,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:40,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +27: [2023-05-13 09:38:40,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:40,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:40,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +25: [2023-05-13 09:38:40,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +25: [2023-05-13 09:38:40,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +25: [2023-05-13 09:38:40,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +15: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:40,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:40,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:40,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:40,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:40,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:40,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:40,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:40,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:40,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:40,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:40,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:40,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:40,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:40,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:40,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:40,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:40,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:40,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:40,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:40,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:40,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +25: [2023-05-13 09:38:40,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:40,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:40,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:40,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +26: [2023-05-13 09:38:40,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:40,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:40,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:40,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:40,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:40,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:40,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:40,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:40,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:40,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +26: [2023-05-13 09:38:40,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:40,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:40,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:40,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:40,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:40,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:40,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +15: [2023-05-13 09:38:40,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +15: [2023-05-13 09:38:40,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +22: [2023-05-13 09:38:40,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +28: [2023-05-13 09:38:40,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +28: [2023-05-13 09:38:40,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:40,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:40,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:40,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:40,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +15: [2023-05-13 09:38:40,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +28: [2023-05-13 09:38:40,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:40,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:40,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:40,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:40,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:40,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:40,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:40,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:40,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:40,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:40,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:40,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +28: [2023-05-13 09:38:40,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:40,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:40,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:40,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 9: [2023-05-13 09:38:40,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:40,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:40,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:40,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:40,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +25: [2023-05-13 09:38:40,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +22: [2023-05-13 09:38:40,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:40,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:40,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:40,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:40,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:40,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:40,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:40,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +31: [2023-05-13 09:38:40,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 1: [2023-05-13 09:38:40,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +31: [2023-05-13 09:38:40,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +31: [2023-05-13 09:38:40,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:40,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:40,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:40,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:40,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:40,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:40,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:40,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:40,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:40,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:40,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:40,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:40,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:40,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:40,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:40,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:40,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:40,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +31: [2023-05-13 09:38:40,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:40,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:40,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:40,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:40,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:40,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt... +24: [2023-05-13 09:38:40,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:40,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:40,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:40,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:40,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:40,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:40,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:40,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:40,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:40,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:40,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:40,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:40,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:40,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:40,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:40,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:40,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:40,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:40,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:40,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +24: [2023-05-13 09:38:40,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:40,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:40,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:40,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:40,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:40,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:40,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:40,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:40,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:40,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:40,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:40,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:40,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:40,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +24: [2023-05-13 09:38:40,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +24: [2023-05-13 09:38:40,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:40,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:40,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:40,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:40,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:40,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:40,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +14: [2023-05-13 09:38:40,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:40,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:40,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +14: [2023-05-13 09:38:40,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +14: [2023-05-13 09:38:40,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +14: [2023-05-13 09:38:40,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:40,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:40,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:40,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:40,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:40,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 0: [2023-05-13 09:38:40,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 0: [2023-05-13 09:38:40,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:40,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:40,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:40,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:40,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:40,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:40,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:40,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:40,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:40,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:40,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:40,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:40,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:40,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:40,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:40,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:40,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:40,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:40,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:40,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:40,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:40,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:40,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:40,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:40,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:41,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:41,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:41,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:41,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:41,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:41,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:41,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:41,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:41,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:41,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:41,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:41,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:41,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:41,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:41,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:41,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:41,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:41,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:41,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:41,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:41,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:41,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:41,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:41,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:41,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:41,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:41,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:41,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:41,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:41,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:41,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:41,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:41,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:41,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:41,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:41,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:41,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:41,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:41,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:41,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:41,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:41,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:41,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:41,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:41,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:41,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:41,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:41,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:41,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:41,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:41,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:41,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:41,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:41,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:41,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:41,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:41,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:41,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:41,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:41,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +19: [2023-05-13 09:38:41,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:41,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +19: [2023-05-13 09:38:41,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:41,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:41,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:41,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:41,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:41,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:41,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:41,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:41,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +11: [2023-05-13 09:38:41,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:41,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:41,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:41,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:41,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:41,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:41,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:41,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:41,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:41,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:41,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:41,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:41,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:41,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:41,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:41,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:41,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:41,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:41,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:41,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:41,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:41,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:41,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:41,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:41,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:41,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:41,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:41,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:41,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:41,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:41,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:41,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:41,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +19: [2023-05-13 09:38:41,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +13: [2023-05-13 09:38:41,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +13: [2023-05-13 09:38:41,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +13: [2023-05-13 09:38:41,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +13: [2023-05-13 09:38:41,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:41,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +10: [2023-05-13 09:38:41,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:41,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:41,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:41,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:41,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:41,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:41,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:41,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:41,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:41,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:41,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:41,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:41,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:41,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:41,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:41,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:41,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:41,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:41,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:41,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:41,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:41,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:41,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:41,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:41,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:41,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:41,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:41,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:41,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:41,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:41,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:41,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:41,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +10: [2023-05-13 09:38:41,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +10: [2023-05-13 09:38:41,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +10: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +27: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:41,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:41,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:41,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +12: [2023-05-13 09:38:41,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +12: [2023-05-13 09:38:41,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +12: [2023-05-13 09:38:41,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +12: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +27: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +13: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:41,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:41,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:41,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:41,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:41,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +21: [2023-05-13 09:38:41,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:41,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:41,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:41,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:41,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:41,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +13: [2023-05-13 09:38:41,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:41,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:41,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:41,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:41,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:41,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:41,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:41,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:41,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +30: [2023-05-13 09:38:41,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +30: [2023-05-13 09:38:41,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +30: [2023-05-13 09:38:41,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:41,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:41,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:41,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:41,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +29: [2023-05-13 09:38:41,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +29: [2023-05-13 09:38:41,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +29: [2023-05-13 09:38:41,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +29: [2023-05-13 09:38:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:41,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +11: [2023-05-13 09:38:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:41,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:41,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:41,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +17: [2023-05-13 09:38:41,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +17: [2023-05-13 09:38:41,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +17: [2023-05-13 09:38:41,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:41,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:41,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +17: [2023-05-13 09:38:41,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +14: [2023-05-13 09:38:41,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:41,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:41,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:41,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:41,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:41,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:41,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:41,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +10: [2023-05-13 09:38:41,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:41,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:41,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +26: [2023-05-13 09:38:41,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:41,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:41,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:41,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +19: [2023-05-13 09:38:41,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:41,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:41,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:41,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:41,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:41,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:41,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:41,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:41,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:41,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:41,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:41,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:41,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:41,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:41,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:41,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:41,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:41,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:41,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +30: [2023-05-13 09:38:41,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:41,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:41,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:41,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:41,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:41,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:41,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:41,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:41,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:41,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:41,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:41,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:41,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:41,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:41,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:41,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:41,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:41,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:41,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:41,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:41,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:41,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:41,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:41,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:41,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:41,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +11: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:41,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:41,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:41,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:41,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:41,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:41,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:41,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:41,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:41,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:41,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:41,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:41,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:41,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:41,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:41,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:41,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:41,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:41,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +12: [2023-05-13 09:38:41,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +10: [2023-05-13 09:38:41,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:41,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:41,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:41,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:41,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +17: [2023-05-13 09:38:41,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:41,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:41,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:41,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:41,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:41,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:41,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:41,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:41,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:41,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +28: [2023-05-13 09:38:41,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:41,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:41,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:41,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:41,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:41,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:41,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:41,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:41,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:41,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:41,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:41,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:41,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:41,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 8: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:41,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +28: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:41,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:41,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:41,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +21: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +30: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +12: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:41,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:41,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +30: [2023-05-13 09:38:41,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +10: [2023-05-13 09:38:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:41,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:41,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:41,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:41,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:41,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:41,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:41,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +10: [2023-05-13 09:38:41,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 8: [2023-05-13 09:38:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:41,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:41,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:41,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +10: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:41,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +28: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:41,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +11: [2023-05-13 09:38:41,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +30: [2023-05-13 09:38:41,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:41,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:41,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +30: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:41,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:41,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +27: [2023-05-13 09:38:41,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:41,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +27: [2023-05-13 09:38:41,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +22: [2023-05-13 09:38:41,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:41,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:41,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:41,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +26: [2023-05-13 09:38:41,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:41,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +22: [2023-05-13 09:38:41,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +30: [2023-05-13 09:38:41,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:41,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +12: [2023-05-13 09:38:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:41,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +21: [2023-05-13 09:38:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:41,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:41,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +26: [2023-05-13 09:38:41,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +29: [2023-05-13 09:38:41,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:41,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:41,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:41,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:41,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:41,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:41,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:41,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:41,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:41,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:41,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:41,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:41,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:41,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:41,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:41,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:41,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:41,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:41,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:41,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:41,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:41,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:41,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:41,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:41,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:41,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:41,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:41,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:41,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:41,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:41,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:41,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:41,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:41,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:41,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:41,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:41,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:41,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:41,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:41,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:41,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:41,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:41,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:41,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:41,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:41,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:41,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:41,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:41,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:41,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:41,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:41,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:41,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +21: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +24: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:41,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +24: [2023-05-13 09:38:41,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:41,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:41,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:41,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:41,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:41,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:41,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:41,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +14: [2023-05-13 09:38:41,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +14: [2023-05-13 09:38:41,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:41,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:41,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:41,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:41,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:41,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:41,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:41,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:41,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:41,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:41,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 0: [2023-05-13 09:38:41,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 0: [2023-05-13 09:38:41,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:41,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:41,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:41,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +15: [2023-05-13 09:38:41,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +15: [2023-05-13 09:38:41,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:41,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:41,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:41,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +24: [2023-05-13 09:38:41,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:41,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:41,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:41,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:41,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:41,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:41,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:41,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:41,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:41,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:41,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:41,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:41,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:41,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:41,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:41,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:41,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:41,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:41,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:41,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:41,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:41,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:41,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:41,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:41,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:41,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:41,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:41,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:41,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:41,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:41,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +16: [2023-05-13 09:38:41,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:41,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:41,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:41,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +31: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +31: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:41,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:41,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +15: [2023-05-13 09:38:41,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +18: [2023-05-13 09:38:41,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +31: [2023-05-13 09:38:41,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:41,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +31: [2023-05-13 09:38:41,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:41,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:41,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:41,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:41,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:41,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:41,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:41,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:41,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:41,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... +24: [2023-05-13 09:38:41,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +16: [2023-05-13 09:38:41,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +15: [2023-05-13 09:38:41,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:41,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:41,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:41,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:41,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:41,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +18: [2023-05-13 09:38:41,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:41,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:41,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:41,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:41,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:41,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:41,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:41,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:41,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:41,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:41,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:41,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:41,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:41,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:41,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:41,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +13: [2023-05-13 09:38:41,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:41,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:41,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:41,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:41,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:41,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:41,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:41,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:41,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:41,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:41,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:41,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:41,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:41,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:41,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:41,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:41,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:41,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:41,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:41,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:41,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:41,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:41,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:41,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:41,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:41,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:41,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:41,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:41,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:41,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:41,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:41,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:41,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:41,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:41,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:41,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:41,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:41,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:41,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:41,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:41,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:41,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:41,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:41,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:41,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:41,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:41,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:41,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:41,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:41,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:41,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +23: [2023-05-13 09:38:41,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:41,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:41,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:41,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:41,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:41,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:41,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:41,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:41,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:41,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:41,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:41,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:41,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:41,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:41,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:41,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:41,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:41,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:41,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:41,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:41,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:41,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:41,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:41,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:41,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:41,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:41,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:41,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:41,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +13: [2023-05-13 09:38:41,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +29: [2023-05-13 09:38:41,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:41,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:41,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:41,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:41,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:41,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:41,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:41,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:41,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:41,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:41,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:41,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:41,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:41,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:41,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:41,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:41,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:41,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:41,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:41,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:41,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:41,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:41,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:41,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +25: [2023-05-13 09:38:41,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +25: [2023-05-13 09:38:41,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +25: [2023-05-13 09:38:41,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:41,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:41,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:41,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:41,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:41,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:41,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:41,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:41,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:41,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +12: [2023-05-13 09:38:41,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:41,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:41,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +21: [2023-05-13 09:38:41,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:41,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:41,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:41,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:41,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +12: [2023-05-13 09:38:41,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:41,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:41,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:41,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:41,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:41,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:41,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +19: [2023-05-13 09:38:41,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +19: [2023-05-13 09:38:41,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:41,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:41,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:41,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:41,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +19: [2023-05-13 09:38:41,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:41,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:41,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:41,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:41,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:41,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:41,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:41,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:41,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:41,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:41,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:41,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +26: [2023-05-13 09:38:41,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:41,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:41,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:41,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:41,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:41,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:41,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:41,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +26: [2023-05-13 09:38:41,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:41,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:41,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:41,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:41,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:41,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:41,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:41,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:41,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:41,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:41,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:41,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:41,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:41,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:41,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:41,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:41,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:41,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:41,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:41,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:41,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:41,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:41,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:41,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:41,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:41,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:41,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:41,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:41,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:41,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:41,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:41,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:41,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:41,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:41,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:41,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:41,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:41,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:41,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:41,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:41,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:41,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:41,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:41,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:41,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:41,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:41,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:41,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:41,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:41,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:41,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:41,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:41,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:41,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:41,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:41,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +13: [2023-05-13 09:38:41,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:41,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:41,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:41,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:41,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:41,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +26: [2023-05-13 09:38:41,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:41,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +26: [2023-05-13 09:38:41,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:41,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:41,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:41,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:41,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:41,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:41,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:41,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:41,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:41,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:41,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:41,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:41,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:41,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:41,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:41,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +26: [2023-05-13 09:38:41,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:41,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:41,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:41,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:41,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:41,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:41,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:41,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:41,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:41,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:41,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:41,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:41,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +26: [2023-05-13 09:38:41,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:41,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:41,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:41,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +28: [2023-05-13 09:38:41,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:41,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +21: [2023-05-13 09:38:41,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +23: [2023-05-13 09:38:41,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +23: [2023-05-13 09:38:41,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:41,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:41,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:41,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:41,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:41,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:41,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:41,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:41,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:41,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:41,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:41,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:41,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +29: [2023-05-13 09:38:41,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +29: [2023-05-13 09:38:41,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +26: [2023-05-13 09:38:41,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:41,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +13: [2023-05-13 09:38:41,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:41,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:41,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:41,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:41,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:41,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:41,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:41,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:41,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:41,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:41,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:41,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:41,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:41,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:41,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:41,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:41,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +30: [2023-05-13 09:38:41,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:41,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:41,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +30: [2023-05-13 09:38:41,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:41,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:41,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +27: [2023-05-13 09:38:41,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +27: [2023-05-13 09:38:41,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +27: [2023-05-13 09:38:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +10: [2023-05-13 09:38:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +10: [2023-05-13 09:38:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +10: [2023-05-13 09:38:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +10: [2023-05-13 09:38:41,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:41,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +13: [2023-05-13 09:38:41,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:41,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:41,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:41,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:41,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:41,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:41,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:41,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:41,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:41,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +27: [2023-05-13 09:38:41,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:41,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:41,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +13: [2023-05-13 09:38:41,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +29: [2023-05-13 09:38:41,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:41,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:41,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:41,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 2: [2023-05-13 09:38:41,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:41,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +17: [2023-05-13 09:38:41,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:41,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:41,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:41,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:41,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:41,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:41,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:41,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:41,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:41,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:41,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:41,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:41,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:41,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:41,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:41,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:41,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:41,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:41,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:41,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:41,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:41,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:41,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:41,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:41,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +31: [2023-05-13 09:38:41,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:41,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:41,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:41,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:41,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:41,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:41,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:41,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +26: [2023-05-13 09:38:41,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +26: [2023-05-13 09:38:41,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +31: [2023-05-13 09:38:41,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:41,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:41,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:41,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +31: [2023-05-13 09:38:41,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:41,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:41,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:41,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:41,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:41,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:41,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +31: [2023-05-13 09:38:41,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +31: [2023-05-13 09:38:41,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +31: [2023-05-13 09:38:41,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:41,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +22: [2023-05-13 09:38:41,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +22: [2023-05-13 09:38:41,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +22: [2023-05-13 09:38:41,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:41,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:41,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +30: [2023-05-13 09:38:41,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:41,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:41,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:41,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:41,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:41,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:41,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:41,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:41,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:41,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:41,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:41,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:41,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:41,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:41,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:41,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:41,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:41,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:41,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:41,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:41,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:41,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:41,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:41,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:41,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:41,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:41,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 2: [2023-05-13 09:38:41,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +29: [2023-05-13 09:38:41,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +31: [2023-05-13 09:38:41,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:41,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:41,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:41,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:41,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:41,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:41,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:41,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:41,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:41,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:41,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:41,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:41,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:41,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:41,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +27: [2023-05-13 09:38:41,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:41,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:41,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +29: [2023-05-13 09:38:41,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 0: [2023-05-13 09:38:41,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 0: [2023-05-13 09:38:41,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 0: [2023-05-13 09:38:41,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:41,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:41,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:41,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:41,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:41,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:41,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:41,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:41,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:41,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:41,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:41,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:41,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +29: [2023-05-13 09:38:41,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:41,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:41,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:41,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:41,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:41,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +30: [2023-05-13 09:38:41,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +27: [2023-05-13 09:38:41,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:41,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:41,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:41,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:41,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:41,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:41,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:41,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:41,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:41,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:41,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +12: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:41,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:41,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:41,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:41,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:41,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:41,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:41,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:41,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:41,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:41,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +29: [2023-05-13 09:38:41,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:41,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:41,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:41,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:41,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:41,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:41,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:41,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:41,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:41,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:41,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:41,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +18: [2023-05-13 09:38:41,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +18: [2023-05-13 09:38:41,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:41,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:41,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +18: [2023-05-13 09:38:41,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +19: [2023-05-13 09:38:41,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:41,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:41,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:41,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:41,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:41,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:41,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:41,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +10: [2023-05-13 09:38:41,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:41,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:41,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:41,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:41,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:41,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:41,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:41,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:41,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:41,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:41,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:41,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:41,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:41,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:41,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:41,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:41,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:41,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:41,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 9: [2023-05-13 09:38:41,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:41,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:41,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:41,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:41,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:41,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:41,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:41,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:41,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:41,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:41,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:41,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:41,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:41,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:41,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:41,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:41,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:41,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:41,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:41,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +31: [2023-05-13 09:38:41,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:41,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:41,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:41,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:41,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +31: [2023-05-13 09:38:41,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:41,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:41,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:41,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:41,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:41,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:41,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +18: [2023-05-13 09:38:41,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:41,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:41,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:41,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:41,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:41,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:41,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:41,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:41,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:41,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:41,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:41,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:41,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:41,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:41,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:41,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:41,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:41,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:41,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:41,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:41,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:41,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:41,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:41,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:41,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:41,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:41,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:41,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:41,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:41,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:41,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:41,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:41,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:41,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:41,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:41,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:41,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:41,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:41,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:41,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:41,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:41,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:41,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:41,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:41,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:41,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:41,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:41,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:41,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:41,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:41,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:41,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:41,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:41,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:41,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +29: [2023-05-13 09:38:41,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:41,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:41,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:41,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:41,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:41,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:41,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:41,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:41,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 2: [2023-05-13 09:38:41,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:41,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:41,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:41,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:41,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:41,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:41,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:41,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:41,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:41,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:41,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:41,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:41,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:41,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:41,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:41,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:41,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:41,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:41,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:41,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:41,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:41,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:41,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:41,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:41,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:41,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:41,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +24: [2023-05-13 09:38:41,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:41,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:41,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:41,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:41,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:41,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:41,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:41,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:41,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:41,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:41,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:41,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:41,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:41,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 9: [2023-05-13 09:38:41,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:41,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:41,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:41,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:41,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:41,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +20: [2023-05-13 09:38:41,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +20: [2023-05-13 09:38:41,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:41,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:41,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:41,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:41,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:41,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:41,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:41,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:41,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:41,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +16: [2023-05-13 09:38:41,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:41,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:41,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:41,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:41,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:41,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:41,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:41,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:41,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:41,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:41,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:41,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:41,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:41,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:41,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:41,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:41,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:41,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:41,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:41,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:41,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:41,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:41,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:41,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:41,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:41,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:41,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:41,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:41,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:41,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:41,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:41,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:41,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:41,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:41,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:41,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:41,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:41,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:41,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:41,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:41,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:41,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:41,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:41,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:41,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:41,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:41,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:41,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:41,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:41,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:41,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:41,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:41,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:41,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:41,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:41,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:41,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:41,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:41,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:41,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:41,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:41,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:41,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:41,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:41,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:41,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:41,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:41,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:41,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:41,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:41,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:41,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:41,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:41,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:41,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:41,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +23: [2023-05-13 09:38:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +23: [2023-05-13 09:38:41,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:41,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:41,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:41,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:41,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:41,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:41,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:41,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:41,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:41,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:41,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:41,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +15: [2023-05-13 09:38:41,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:41,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:41,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:41,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:41,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:41,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:41,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:41,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +15: [2023-05-13 09:38:41,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:41,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:41,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:41,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:41,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_30-model_01-model_states.pt. +17: [2023-05-13 09:38:41,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:41,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:41,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:41,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:41,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:41,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:41,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:41,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:41,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:41,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:41,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:41,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:41,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:41,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:41,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:41,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:41,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:41,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:41,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:41,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:41,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:41,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:41,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +25: [2023-05-13 09:38:41,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +27: [2023-05-13 09:38:41,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:41,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:41,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +25: [2023-05-13 09:38:41,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +13: [2023-05-13 09:38:41,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:41,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:41,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:41,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:41,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:41,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:41,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:41,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:41,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:41,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:41,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:41,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:41,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:41,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:41,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:41,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:41,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:41,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:41,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:41,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:41,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:41,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:41,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +19: [2023-05-13 09:38:41,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:41,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 3: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:41,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +29: [2023-05-13 09:38:41,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +29: [2023-05-13 09:38:41,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +29: [2023-05-13 09:38:41,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:41,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:41,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:41,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:41,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:41,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:41,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +21: [2023-05-13 09:38:41,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +21: [2023-05-13 09:38:41,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:41,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:41,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:41,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +27: [2023-05-13 09:38:41,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +27: [2023-05-13 09:38:41,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +21: [2023-05-13 09:38:41,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:41,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:41,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:41,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +15: [2023-05-13 09:38:41,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:41,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:41,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +27: [2023-05-13 09:38:41,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:41,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +28: [2023-05-13 09:38:41,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +28: [2023-05-13 09:38:41,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +15: [2023-05-13 09:38:41,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +12: [2023-05-13 09:38:41,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:41,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +11: [2023-05-13 09:38:41,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:41,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:41,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:41,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:41,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:41,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:41,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:41,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:41,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:41,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:41,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:41,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:41,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:41,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:41,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:41,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:41,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:41,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:41,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:41,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:41,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:41,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:41,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +15: [2023-05-13 09:38:41,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:41,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:41,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:41,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +15: [2023-05-13 09:38:41,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:41,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:41,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:41,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:41,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:41,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:41,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:41,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:41,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:41,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:41,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:41,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:41,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:41,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:41,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:41,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:41,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:41,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:41,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:41,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:41,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:41,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:41,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:41,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:41,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:41,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:41,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:41,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +26: [2023-05-13 09:38:41,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:41,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:41,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:41,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:41,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:41,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +25: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:41,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:41,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:41,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:41,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:41,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:41,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:41,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:41,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:41,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:41,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:41,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:41,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:41,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:41,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:41,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:41,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +28: [2023-05-13 09:38:41,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:41,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:41,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:41,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:41,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:41,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:41,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:41,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:41,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:41,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:41,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +31: [2023-05-13 09:38:41,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:41,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:41,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:41,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:41,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:41,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:41,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +31: [2023-05-13 09:38:41,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:41,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:41,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:41,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:41,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:41,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:41,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:41,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:41,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:41,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:41,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:41,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:41,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:41,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:41,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +23: [2023-05-13 09:38:41,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:41,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +19: [2023-05-13 09:38:41,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:41,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:41,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:41,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:41,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +19: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:41,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +10: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:41,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +23: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +22: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:41,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:41,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:41,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:41,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:41,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:41,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:41,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:41,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:41,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:41,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:41,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +31: [2023-05-13 09:38:41,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:41,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:41,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:41,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:41,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:41,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:41,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:41,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:41,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:41,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +29: [2023-05-13 09:38:41,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +29: [2023-05-13 09:38:41,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:41,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:41,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:41,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 7: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:41,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:41,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +26: [2023-05-13 09:38:41,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:41,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:41,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:41,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:41,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:41,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:41,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:41,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:41,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:41,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:41,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:41,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:41,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:41,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:41,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:41,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:41,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:41,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:41,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:41,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:41,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:41,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:41,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:41,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:41,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:41,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:41,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:41,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:41,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:41,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:41,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:41,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:41,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:41,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +10: [2023-05-13 09:38:41,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:41,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:41,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:41,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:41,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:41,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:41,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:41,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:41,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:41,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:41,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:41,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:41,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:41,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:41,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:41,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:41,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:41,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:41,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:41,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:41,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:41,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:41,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:41,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:41,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:41,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:41,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:41,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:41,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:41,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:41,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:41,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +26: [2023-05-13 09:38:41,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:41,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:41,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:41,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:41,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +26: [2023-05-13 09:38:41,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:41,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:41,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:41,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +10: [2023-05-13 09:38:41,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:41,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:41,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:41,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:41,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:41,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:41,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:41,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:41,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:41,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:41,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 6: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:41,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:41,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:41,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:41,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:41,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:41,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:41,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:41,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:41,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:41,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:41,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:41,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:41,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:41,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:41,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:41,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:41,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:41,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:41,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:41,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:41,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:41,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:41,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:41,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:41,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:41,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:41,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:41,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:41,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:41,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:41,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:41,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:41,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:41,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:41,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:41,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:41,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:41,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:41,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:41,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +24: [2023-05-13 09:38:41,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:41,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:41,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:41,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:41,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:41,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +24: [2023-05-13 09:38:41,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:41,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +24: [2023-05-13 09:38:41,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:41,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:41,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:41,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:41,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:41,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:41,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:41,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:41,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:41,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:41,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:41,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:41,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:41,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:41,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:41,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:41,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:41,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:41,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:41,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:41,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:41,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:41,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:41,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:41,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:41,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:41,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:41,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:41,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:41,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:41,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:41,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:41,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:41,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:41,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:41,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:41,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:41,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:41,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:41,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:41,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +31: [2023-05-13 09:38:41,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +22: [2023-05-13 09:38:41,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:41,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:41,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +26: [2023-05-13 09:38:41,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:41,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:41,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:41,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:41,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +26: [2023-05-13 09:38:41,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:41,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:41,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:41,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:41,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:41,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:41,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:41,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +12: [2023-05-13 09:38:41,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:41,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:41,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:41,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:41,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:41,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:41,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +17: [2023-05-13 09:38:41,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:41,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:41,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:41,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:41,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:41,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:41,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:41,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:41,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:41,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:41,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +31: [2023-05-13 09:38:41,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +15: [2023-05-13 09:38:41,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:41,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:41,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:41,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:41,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:41,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:41,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:41,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:41,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:41,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:41,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:41,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:41,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:41,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:41,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:41,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:41,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:41,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:41,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:41,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:41,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:41,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:41,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:41,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:41,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:41,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:41,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:41,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:41,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:41,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:41,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:41,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:41,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +22: [2023-05-13 09:38:41,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:41,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:41,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:41,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:41,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:41,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:41,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:41,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:41,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:41,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:41,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:41,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:41,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:41,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:41,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:41,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:41,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:41,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:41,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:41,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:41,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:41,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:41,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 3: [2023-05-13 09:38:41,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:41,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:41,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:41,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:41,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +22: [2023-05-13 09:38:41,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:41,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:41,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:41,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:41,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:41,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:41,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:41,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:41,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 8: [2023-05-13 09:38:41,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:41,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +19: [2023-05-13 09:38:41,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:41,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:41,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:41,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +31: [2023-05-13 09:38:41,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:42,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:42,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +15: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:42,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:42,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +31: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:42,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +31: [2023-05-13 09:38:42,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:42,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:42,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:42,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +31: [2023-05-13 09:38:42,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:42,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:42,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:42,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +11: [2023-05-13 09:38:42,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:42,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +15: [2023-05-13 09:38:42,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:42,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:42,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:42,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +14: [2023-05-13 09:38:42,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +31: [2023-05-13 09:38:42,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:42,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:42,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +15: [2023-05-13 09:38:42,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_31-model_01-model_states.pt. +24: [2023-05-13 09:38:42,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:42,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:42,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:42,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:42,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:42,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:42,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:42,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:42,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:42,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:42,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:42,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:42,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:42,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:42,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +20: [2023-05-13 09:38:42,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +20: [2023-05-13 09:38:42,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:42,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:42,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:42,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:42,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:42,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:42,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:42,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:42,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:42,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +17: [2023-05-13 09:38:42,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:42,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:42,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:42,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:42,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:42,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:42,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:42,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:42,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:42,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:42,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:42,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:42,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:42,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:42,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:42,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:42,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:42,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:42,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:42,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:42,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:42,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:42,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:42,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:42,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:42,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:42,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:42,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:42,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:42,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +17: [2023-05-13 09:38:42,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:42,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:42,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:42,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:42,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:42,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:42,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:42,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +11: [2023-05-13 09:38:42,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +21: [2023-05-13 09:38:42,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:42,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +27: [2023-05-13 09:38:42,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:42,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +27: [2023-05-13 09:38:42,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:42,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:42,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:42,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:42,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:42,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:42,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:42,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:42,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:42,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:42,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:42,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:42,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:42,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:42,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:42,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:42,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:42,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:42,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:42,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +18: [2023-05-13 09:38:42,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:42,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:42,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:42,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +18: [2023-05-13 09:38:42,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +15: [2023-05-13 09:38:42,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:42,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:42,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:42,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:42,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:42,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:42,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:42,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:42,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +11: [2023-05-13 09:38:42,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:42,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:42,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:42,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:42,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:42,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:42,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:42,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:42,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:42,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:42,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:42,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:42,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:42,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:42,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:42,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:42,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:42,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:42,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:42,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +18: [2023-05-13 09:38:42,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:42,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:42,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:42,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:42,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:42,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:42,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:42,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:42,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:42,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:42,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:42,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +20: [2023-05-13 09:38:42,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:42,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +13: [2023-05-13 09:38:42,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +20: [2023-05-13 09:38:42,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:42,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:42,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:42,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:42,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +13: [2023-05-13 09:38:42,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +26: [2023-05-13 09:38:42,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:42,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:42,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:42,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:42,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:42,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +21: [2023-05-13 09:38:42,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:42,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:42,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:42,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:42,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:42,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:42,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:42,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:42,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:42,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:42,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:42,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:42,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:42,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:42,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:42,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +10: [2023-05-13 09:38:42,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +10: [2023-05-13 09:38:42,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:42,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:42,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:42,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +13: [2023-05-13 09:38:42,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:42,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +11: [2023-05-13 09:38:42,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:42,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:42,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:42,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:42,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:42,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +11: [2023-05-13 09:38:42,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +11: [2023-05-13 09:38:42,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:42,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:42,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:42,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:42,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:42,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:42,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:42,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:42,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:42,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:42,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +18: [2023-05-13 09:38:42,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +10: [2023-05-13 09:38:42,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:42,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:42,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +21: [2023-05-13 09:38:42,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:42,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:42,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:42,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:42,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:42,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:42,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:42,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:42,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:42,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:42,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:42,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:42,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:42,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +27: [2023-05-13 09:38:42,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:42,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:42,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:42,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:42,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:42,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:42,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:42,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:42,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +12: [2023-05-13 09:38:42,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:42,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:42,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:42,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:42,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:42,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:42,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:42,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +29: [2023-05-13 09:38:42,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:42,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:42,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:42,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:42,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:42,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:42,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:42,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:42,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:42,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:42,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:42,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:42,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:42,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:42,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +23: [2023-05-13 09:38:42,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:42,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +28: [2023-05-13 09:38:42,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:42,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:42,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:42,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +28: [2023-05-13 09:38:42,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +29: [2023-05-13 09:38:42,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +28: [2023-05-13 09:38:42,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:42,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:42,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:42,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:42,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:42,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:42,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:42,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:42,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:42,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:42,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +10: [2023-05-13 09:38:42,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:42,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +23: [2023-05-13 09:38:42,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +29: [2023-05-13 09:38:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +10: [2023-05-13 09:38:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:42,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:42,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:42,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:42,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:42,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:42,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:42,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:42,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:42,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:42,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:42,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:42,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +18: [2023-05-13 09:38:42,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:42,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:42,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:42,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:42,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:42,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:42,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:42,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:42,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:42,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:42,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +10: [2023-05-13 09:38:42,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:42,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:42,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:42,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:42,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +12: [2023-05-13 09:38:42,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:42,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:42,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:42,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:42,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:42,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +30: [2023-05-13 09:38:42,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +30: [2023-05-13 09:38:42,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:42,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:42,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:42,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:42,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:42,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:42,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +27: [2023-05-13 09:38:42,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:42,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:42,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:42,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:42,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:42,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:42,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:42,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +28: [2023-05-13 09:38:42,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:42,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:42,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +25: [2023-05-13 09:38:42,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +25: [2023-05-13 09:38:42,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +25: [2023-05-13 09:38:42,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:42,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:42,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:42,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:42,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:42,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:42,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:42,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:42,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:42,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:42,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:42,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:42,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:42,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:42,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +12: [2023-05-13 09:38:42,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:42,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:42,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:42,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:42,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:42,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:42,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:42,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:42,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:42,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:42,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:42,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:42,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:42,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:42,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:42,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:42,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:42,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:42,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:42,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:42,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:42,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:42,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:42,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:42,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:42,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:42,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:42,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:42,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:42,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:42,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:42,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:42,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:42,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:42,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:42,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:42,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:42,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:42,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +28: [2023-05-13 09:38:42,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +18: [2023-05-13 09:38:42,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:42,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:42,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:42,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:42,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:42,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:42,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:42,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +10: [2023-05-13 09:38:42,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +28: [2023-05-13 09:38:42,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:42,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:42,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:42,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:42,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:42,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +28: [2023-05-13 09:38:42,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:42,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:42,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:42,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +30: [2023-05-13 09:38:42,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:42,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:42,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +28: [2023-05-13 09:38:42,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:42,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:42,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:42,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:42,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:42,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:42,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +15: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +25: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:42,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:42,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:42,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:42,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. +25: [2023-05-13 09:38:42,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:42,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +31: [2023-05-13 09:38:42,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:42,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:42,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:42,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:42,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:42,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:42,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:42,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:42,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:42,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:42,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:42,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:42,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:42,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:42,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +25: [2023-05-13 09:38:42,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:42,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:42,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:42,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:42,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:42,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:42,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:42,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:42,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:42,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:42,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:42,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +10: [2023-05-13 09:38:42,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:42,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:42,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:42,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:42,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:42,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:42,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:42,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:42,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:42,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:42,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:42,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:42,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:42,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:42,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:42,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:42,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:42,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:42,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:42,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:42,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:42,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:42,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:42,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:42,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:42,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:42,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:42,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:42,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:42,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:42,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:42,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:42,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:42,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:42,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:42,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:42,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +14: [2023-05-13 09:38:42,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:42,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:42,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +17: [2023-05-13 09:38:42,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:42,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:42,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:42,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:42,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +26: [2023-05-13 09:38:42,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +26: [2023-05-13 09:38:42,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +26: [2023-05-13 09:38:42,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:42,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:42,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:42,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:42,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:42,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:42,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:42,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:42,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:42,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +15: [2023-05-13 09:38:42,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:42,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:42,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:42,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:42,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:42,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +11: [2023-05-13 09:38:42,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +11: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +15: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +19: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +17: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +11: [2023-05-13 09:38:42,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:42,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:42,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:42,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:42,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:42,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:42,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +14: [2023-05-13 09:38:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:42,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:42,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:42,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +15: [2023-05-13 09:38:42,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +17: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +15: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:42,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:42,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +19: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:42,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:42,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:42,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:42,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:42,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:42,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:42,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:42,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:42,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:42,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:42,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:42,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:42,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:42,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:42,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +11: [2023-05-13 09:38:42,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:42,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:42,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:42,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:42,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:42,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:42,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:42,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +26: [2023-05-13 09:38:42,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +26: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +31: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:42,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +11: [2023-05-13 09:38:42,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:42,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:42,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:42,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:42,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:42,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:42,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +26: [2023-05-13 09:38:42,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:42,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +31: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:42,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:42,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:42,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:42,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:42,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +14: [2023-05-13 09:38:42,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:42,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:42,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:42,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:42,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:42,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:42,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:42,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:42,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +11: [2023-05-13 09:38:42,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:42,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:42,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +17: [2023-05-13 09:38:42,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:42,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:42,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:42,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:42,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:42,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:42,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:42,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:42,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:42,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:42,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:42,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:42,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:42,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:42,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:42,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:42,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:42,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:42,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:42,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:42,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:42,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:42,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:42,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:42,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:42,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:42,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:42,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:42,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:42,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:42,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:42,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:42,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:42,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:42,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:42,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:42,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:42,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:42,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:42,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:42,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:42,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:42,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:42,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:42,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:42,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:42,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:42,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:42,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:42,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:42,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:42,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:42,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:42,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:42,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:42,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:42,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:42,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:42,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:42,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:42,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:42,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:42,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:42,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:42,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:42,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:42,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:42,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:42,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:42,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:42,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:42,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:42,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:42,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:42,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:42,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:42,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:42,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:42,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:42,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:42,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:42,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:42,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:42,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:42,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +20: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:42,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:42,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:42,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:42,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:42,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:42,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:42,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:42,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:42,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt... +19: [2023-05-13 09:38:42,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt... +19: [2023-05-13 09:38:42,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt... +19: [2023-05-13 09:38:42,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:38:42,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:42,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:42,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:42,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:42,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:42,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:42,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:42,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:42,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:42,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:42,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:42,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:42,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt... +29: [2023-05-13 09:38:42,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt... +29: [2023-05-13 09:38:42,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt... +29: [2023-05-13 09:38:42,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt... +18: [2023-05-13 09:38:42,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:42,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:42,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:42,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:42,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:42,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:42,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:42,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:42,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:42,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:42,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:42,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:42,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:42,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:42,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +23: [2023-05-13 09:38:42,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:42,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:42,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:42,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:42,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:42,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:42,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:42,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:42,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:42,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:42,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:42,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +24: [2023-05-13 09:38:42,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +24: [2023-05-13 09:38:42,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:42,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt... +20: [2023-05-13 09:38:42,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt... +12: [2023-05-13 09:38:42,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:42,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:42,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:42,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:42,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:42,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt... +20: [2023-05-13 09:38:42,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:38:42,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:38:42,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:38:42,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:38:42,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:38:42,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:42,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:42,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:42,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:42,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:42,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:42,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:42,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:42,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:42,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:42,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:42,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:42,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:42,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:42,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:42,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:42,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:42,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:42,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:42,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:42,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:42,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:42,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:42,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:42,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:42,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:42,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:42,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:42,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:42,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:42,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +23: [2023-05-13 09:38:42,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +23: [2023-05-13 09:38:42,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +23: [2023-05-13 09:38:42,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:42,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:42,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:42,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:42,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:42,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:42,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:42,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:42,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:42,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:42,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:42,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +11: [2023-05-13 09:38:42,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +11: [2023-05-13 09:38:42,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +11: [2023-05-13 09:38:42,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +24: [2023-05-13 09:38:42,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:42,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:42,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:42,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:42,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:42,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:42,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:42,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:42,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:42,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:42,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +25: [2023-05-13 09:38:42,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:38:42,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:38:42,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:38:42,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:38:42,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +23: [2023-05-13 09:38:42,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:42,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:42,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:42,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:42,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:42,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:42,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:42,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:42,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:42,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:42,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:42,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:42,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:42,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:42,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:42,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:42,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:42,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:42,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:42,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +29: [2023-05-13 09:38:42,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +29: [2023-05-13 09:38:42,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +11: [2023-05-13 09:38:42,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:42,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:42,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:42,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +21: [2023-05-13 09:38:42,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +21: [2023-05-13 09:38:42,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +21: [2023-05-13 09:38:42,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +24: [2023-05-13 09:38:42,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:42,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:42,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:42,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +20: [2023-05-13 09:38:42,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:42,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:42,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:42,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:42,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:42,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:42,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:42,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:42,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:42,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:42,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:42,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:42,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:42,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:42,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:42,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:42,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:42,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:42,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:42,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:42,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:42,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:42,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:42,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:42,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:42,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:42,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:42,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:42,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:42,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:42,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:42,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:42,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:42,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:42,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:42,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:42,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:42,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:42,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:42,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:42,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:42,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:42,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:42,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:42,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:42,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:42,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:42,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:42,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +21: [2023-05-13 09:38:42,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:42,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:42,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:42,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:42,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:42,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:42,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:42,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:42,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:42,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:42,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:42,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:42,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:42,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:42,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:42,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:42,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:42,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:42,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:42,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:42,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:42,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:42,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:42,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:42,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:42,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +10: [2023-05-13 09:38:42,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:42,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +11: [2023-05-13 09:38:42,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:42,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:42,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:42,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:42,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:42,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:42,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:42,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:42,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:42,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:42,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:42,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:42,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:42,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:42,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:42,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:42,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:42,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +30: [2023-05-13 09:38:42,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:42,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:42,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:42,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +11: [2023-05-13 09:38:42,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:42,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:42,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +28: [2023-05-13 09:38:42,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:42,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:42,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +28: [2023-05-13 09:38:42,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +28: [2023-05-13 09:38:42,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:42,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +25: [2023-05-13 09:38:42,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +25: [2023-05-13 09:38:42,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +25: [2023-05-13 09:38:42,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:42,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +11: [2023-05-13 09:38:42,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:42,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:42,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +21: [2023-05-13 09:38:42,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:42,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:42,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +12: [2023-05-13 09:38:42,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:42,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:42,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:42,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +17: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:42,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:42,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:42,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:42,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:42,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:42,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +13: [2023-05-13 09:38:42,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +13: [2023-05-13 09:38:42,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:42,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:42,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:42,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:42,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:42,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +15: [2023-05-13 09:38:42,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:42,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:42,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:42,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:42,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:42,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:42,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:42,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:42,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:42,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:42,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:42,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:42,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:42,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:42,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:42,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +25: [2023-05-13 09:38:42,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:42,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:42,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:42,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +22: [2023-05-13 09:38:42,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +25: [2023-05-13 09:38:42,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:42,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +27: [2023-05-13 09:38:42,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:42,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:42,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:42,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:42,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:42,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:42,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:42,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:42,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:42,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:42,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:42,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:42,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:42,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:42,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:42,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:42,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:42,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:42,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:42,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:42,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:42,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:42,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:42,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +27: [2023-05-13 09:38:42,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +30: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 6: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +20: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +18: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. + 1: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:42,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +17: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt... +28: [2023-05-13 09:38:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt... +28: [2023-05-13 09:38:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +29: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +31: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt... + 3: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:42,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:42,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:42,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:42,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:42,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:42,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:42,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:42,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:42,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:42,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:42,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +28: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +29: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +10: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:42,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:42,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:42,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:42,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:42,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:42,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:42,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:42,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:42,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:42,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:42,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:42,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:42,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:42,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:42,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:42,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:42,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:42,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:42,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:42,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:42,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:42,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:42,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:42,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:42,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:42,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +28: [2023-05-13 09:38:42,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:42,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:42,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:42,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:42,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:42,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +14: [2023-05-13 09:38:42,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:42,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:42,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +25: [2023-05-13 09:38:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +25: [2023-05-13 09:38:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:42,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +14: [2023-05-13 09:38:42,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:42,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +28: [2023-05-13 09:38:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:42,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:42,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:42,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:38:42,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:38:42,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:38:42,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:38:42,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:42,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +16: [2023-05-13 09:38:42,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +18: [2023-05-13 09:38:42,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:42,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:42,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:42,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:42,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +18: [2023-05-13 09:38:42,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +30: [2023-05-13 09:38:42,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +30: [2023-05-13 09:38:42,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +30: [2023-05-13 09:38:42,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:42,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:42,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:42,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:42,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:42,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:42,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:42,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:42,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:42,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:42,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +16: [2023-05-13 09:38:42,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:42,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:42,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:42,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +31: [2023-05-13 09:38:42,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:38:42,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:38:42,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:38:42,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt... +12: [2023-05-13 09:38:42,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:42,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:42,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:42,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:42,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:42,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:42,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:42,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:42,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:42,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:42,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:42,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:42,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:42,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:42,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:42,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:42,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:42,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:42,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:42,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:42,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:42,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt... +22: [2023-05-13 09:38:42,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:42,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:42,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:42,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:42,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:42,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:42,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:42,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:42,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:42,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:42,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:42,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:42,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:42,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:42,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:42,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:42,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:42,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:42,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:42,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:42,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:42,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:42,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:42,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:42,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:42,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:42,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:42,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:42,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... +11: [2023-05-13 09:38:42,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:42,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:42,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:42,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:42,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:42,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:42,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:42,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:42,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:42,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:42,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:42,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +15: [2023-05-13 09:38:42,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:42,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt... +22: [2023-05-13 09:38:42,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt... +22: [2023-05-13 09:38:42,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt... +22: [2023-05-13 09:38:42,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:38:42,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:42,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:42,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... + 5: [2023-05-13 09:38:42,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... + 5: [2023-05-13 09:38:42,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... + 5: [2023-05-13 09:38:42,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... + 8: [2023-05-13 09:38:42,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:42,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:42,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:42,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:42,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:42,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:42,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:42,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:42,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:42,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:42,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:42,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:42,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +15: [2023-05-13 09:38:42,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:42,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:42,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:42,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +15: [2023-05-13 09:38:42,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:42,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:42,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:42,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:42,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:42,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:42,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:42,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:42,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:42,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:42,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:42,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:42,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:42,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:42,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:42,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:42,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:42,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:42,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:42,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:42,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:42,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:42,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:42,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:42,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:38:42,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:38:42,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:38:42,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:42,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... + 3: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:42,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:42,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:42,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:42,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:42,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:42,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +23: [2023-05-13 09:38:42,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:42,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:42,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:42,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:42,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +21: [2023-05-13 09:38:42,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:42,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:42,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:42,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:42,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:42,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:42,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:42,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:42,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 9: [2023-05-13 09:38:42,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... + 3: [2023-05-13 09:38:42,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... + 3: [2023-05-13 09:38:42,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... + 3: [2023-05-13 09:38:42,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:38:42,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:42,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +21: [2023-05-13 09:38:42,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:42,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:42,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:42,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:42,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:42,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:42,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... + 8: [2023-05-13 09:38:42,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... + 8: [2023-05-13 09:38:42,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... + 8: [2023-05-13 09:38:42,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... +11: [2023-05-13 09:38:42,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +11: [2023-05-13 09:38:42,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +11: [2023-05-13 09:38:42,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +11: [2023-05-13 09:38:42,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:38:42,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +26: [2023-05-13 09:38:42,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:42,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:42,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:42,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:42,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:42,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:42,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:42,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:42,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:42,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:42,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:42,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:42,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:42,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:42,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:42,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:42,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:42,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:42,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:42,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:42,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:42,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:42,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:42,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:42,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:42,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:42,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:42,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:42,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:42,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:42,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:38:42,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:38:42,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:38:42,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt... +23: [2023-05-13 09:38:42,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:42,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:42,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:42,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:42,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:42,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:42,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:42,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:42,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:42,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:42,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:42,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:42,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:42,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:42,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:42,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:42,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:42,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:42,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:42,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:42,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:42,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:42,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:42,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:42,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:42,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:42,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:42,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:42,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:42,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:42,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:42,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:42,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:42,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:42,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:42,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:42,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:42,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +27: [2023-05-13 09:38:42,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:42,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:42,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:42,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:42,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:42,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:42,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:42,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:42,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:42,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:42,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:42,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:42,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:42,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:42,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 8: [2023-05-13 09:38:42,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:42,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:42,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:42,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:42,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:42,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:42,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:42,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:42,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:42,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt... +23: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt... +23: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt... +14: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt... +26: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:42,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:42,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:42,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:42,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:42,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:42,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:42,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:42,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:42,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:42,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:42,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:42,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:42,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:42,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:42,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:42,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:42,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:42,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:42,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:42,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:42,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:42,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:42,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:42,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:42,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:42,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:42,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:42,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:42,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:42,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:42,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:42,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:42,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:42,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:42,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:42,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:42,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:42,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:42,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:42,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:42,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:42,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:42,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:42,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:42,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:42,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:42,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:42,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:42,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:42,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:42,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:42,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:42,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:42,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:42,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:42,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:42,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:42,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:42,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:42,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:42,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:42,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:42,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:42,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:42,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:42,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:42,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:42,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:42,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:42,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:42,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:42,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:42,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:42,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:42,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:42,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt... +18: [2023-05-13 09:38:42,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt... +18: [2023-05-13 09:38:42,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt... +18: [2023-05-13 09:38:42,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:38:42,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:42,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt... +24: [2023-05-13 09:38:42,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt... +24: [2023-05-13 09:38:42,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt... +24: [2023-05-13 09:38:42,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt... +26: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt... +26: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt... +26: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt... +26: [2023-05-13 09:38:42,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:38:42,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:42,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:42,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:38:42,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:38:42,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:38:42,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:38:42,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +13: [2023-05-13 09:38:42,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:42,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:42,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +22: [2023-05-13 09:38:42,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:42,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:42,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:42,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +13: [2023-05-13 09:38:42,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:42,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:42,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:42,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:42,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:42,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:42,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:38:42,848] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 232 +10: [2023-05-13 09:38:42,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:42,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:42,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:42,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +12: [2023-05-13 09:38:42,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +12: [2023-05-13 09:38:42,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +12: [2023-05-13 09:38:42,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +22: [2023-05-13 09:38:42,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:42,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:42,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:42,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:42,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:42,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:42,855] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 232 + 4: [2023-05-13 09:38:42,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:42,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:42,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:42,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:42,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:42,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +25: [2023-05-13 09:38:42,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:42,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:42,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:42,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:42,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 7: [2023-05-13 09:38:42,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:42,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:42,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +13: [2023-05-13 09:38:42,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:42,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:42,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:42,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:42,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:42,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +26: [2023-05-13 09:38:42,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +26: [2023-05-13 09:38:42,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +26: [2023-05-13 09:38:42,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:42,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:42,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:42,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:42,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:42,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +17: [2023-05-13 09:38:42,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:42,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:42,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:42,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:42,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:42,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +28: [2023-05-13 09:38:42,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:38:42,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:42,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +10: [2023-05-13 09:38:42,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:42,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:42,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:42,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:42,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:42,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:42,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:42,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:42,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +25: [2023-05-13 09:38:42,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +25: [2023-05-13 09:38:42,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +25: [2023-05-13 09:38:42,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:42,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:42,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:42,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:42,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:42,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:42,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:42,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:42,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:42,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:42,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:42,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:42,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt. + 4: [2023-05-13 09:38:42,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:42,886] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 152 + 4: [2023-05-13 09:38:42,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:42,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:42,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:42,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:42,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:42,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:42,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:42,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:42,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:42,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:42,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:42,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:42,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:42,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:42,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:42,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:42,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:42,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:42,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +28: [2023-05-13 09:38:42,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:42,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:42,892] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 152 + 4: [2023-05-13 09:38:42,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:42,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:42,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:42,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +28: [2023-05-13 09:38:42,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:42,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:42,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:42,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:42,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +13: [2023-05-13 09:38:42,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:42,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:42,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:42,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 4: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:42,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:42,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:42,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:42,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:42,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:42,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:42,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:42,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +19: [2023-05-13 09:38:42,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +19: [2023-05-13 09:38:42,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +10: [2023-05-13 09:38:42,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:42,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +19: [2023-05-13 09:38:42,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +25: [2023-05-13 09:38:42,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:42,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:42,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:42,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:42,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +16: [2023-05-13 09:38:42,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:42,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:42,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:42,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:42,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +16: [2023-05-13 09:38:42,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:42,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:42,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:42,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:42,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:42,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:42,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +25: [2023-05-13 09:38:42,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:42,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:42,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:42,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:42,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:42,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:42,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:42,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:42,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:42,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:42,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:42,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:42,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:42,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:42,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt. + 0: [2023-05-13 09:38:42,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:42,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:42,916] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 166 +25: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 0: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +19: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:42,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:42,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:42,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:42,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:42,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:42,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:42,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:38:42,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:42,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:42,919] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 78 + 7: [2023-05-13 09:38:42,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:38:42,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:38:42,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:38:42,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:42,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:42,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:42,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:42,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:42,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:42,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:42,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:42,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:42,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:42,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:42,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:42,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:42,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:42,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:42,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:42,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 0: > overriding learning rate value to 0.0002 + 0: > overriding minimum learning rate value to 2e-05 + 0: > overriding warmup iterations value to 0 + 0: > overriding total number of iterations value to 1 + 0: > overriding decay style value to cosine +28: [2023-05-13 09:38:42,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:42,925] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 166 + 9: [2023-05-13 09:38:42,925] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 78 +11: [2023-05-13 09:38:42,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:42,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:42,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:42,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:42,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:42,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:42,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:42,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +28: [2023-05-13 09:38:42,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:42,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:42,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:42,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 4: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... + 4: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... + 4: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... + 4: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:42,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:42,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:42,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:42,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:42,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:38:42,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:38:42,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:38:42,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:38:42,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:42,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:42,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:42,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:42,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:42,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:42,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:42,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:42,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:42,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:42,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:42,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:42,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:42,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:42,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:42,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:42,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:42,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:42,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:42,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:42,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:42,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:42,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:42,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:42,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:42,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:42,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +17: [2023-05-13 09:38:42,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:42,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:42,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:42,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:42,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:42,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:42,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:42,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:42,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:42,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:42,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:42,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:42,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +21: [2023-05-13 09:38:42,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:42,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:42,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:42,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:42,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:42,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:42,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:42,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +17: [2023-05-13 09:38:42,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:42,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:42,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:42,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:42,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:42,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:42,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:42,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... + 0: [2023-05-13 09:38:42,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... + 0: [2023-05-13 09:38:42,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... + 0: [2023-05-13 09:38:42,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... +19: [2023-05-13 09:38:42,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:42,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:42,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:42,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:42,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:42,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:42,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:42,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:42,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:42,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:42,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:42,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +17: [2023-05-13 09:38:42,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:42,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:42,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +17: [2023-05-13 09:38:42,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:42,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:42,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:42,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:42,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:42,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:42,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:42,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:42,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:42,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:42,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:42,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:42,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:42,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:42,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:42,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:42,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:42,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:42,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:42,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:42,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:42,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:42,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:42,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:42,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:42,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:42,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +23: [2023-05-13 09:38:42,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:42,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:42,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:42,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:42,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +23: [2023-05-13 09:38:42,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:42,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +23: [2023-05-13 09:38:42,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 3: [2023-05-13 09:38:42,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:42,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +17: [2023-05-13 09:38:42,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:42,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:42,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:42,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:38:42,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:42,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:42,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:42,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:42,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:42,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 3: [2023-05-13 09:38:42,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +21: [2023-05-13 09:38:42,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:42,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:38:42,979] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 236 +21: [2023-05-13 09:38:42,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:42,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:42,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:42,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +30: [2023-05-13 09:38:42,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +30: [2023-05-13 09:38:42,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:42,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:42,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:42,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +30: [2023-05-13 09:38:42,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:42,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +30: [2023-05-13 09:38:42,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:42,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:42,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:42,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:42,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:42,987] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 236 +27: [2023-05-13 09:38:42,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:42,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +23: [2023-05-13 09:38:42,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:42,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:42,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:42,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:42,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +23: [2023-05-13 09:38:42,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:42,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:42,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +27: [2023-05-13 09:38:42,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:42,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:42,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +27: [2023-05-13 09:38:42,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:42,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:42,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:42,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +15: [2023-05-13 09:38:42,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +15: [2023-05-13 09:38:42,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:42,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:42,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:42,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:42,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:42,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:42,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:42,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +20: [2023-05-13 09:38:42,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:42,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:42,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:42,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +31: [2023-05-13 09:38:42,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:42,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:43,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:43,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:43,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:43,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:43,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:38:43,007] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 230 +15: [2023-05-13 09:38:43,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:43,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:43,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +14: [2023-05-13 09:38:43,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +14: [2023-05-13 09:38:43,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +14: [2023-05-13 09:38:43,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:38:43,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:38:43,012] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 128 +15: [2023-05-13 09:38:43,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:43,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +23: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +28: [2023-05-13 09:38:43,016] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 230 +23: [2023-05-13 09:38:43,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:43,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:43,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:43,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:43,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:43,019] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 128 +23: [2023-05-13 09:38:43,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +27: [2023-05-13 09:38:43,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +20: [2023-05-13 09:38:43,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:38:43,024] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 18 +27: [2023-05-13 09:38:43,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:43,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +30: [2023-05-13 09:38:43,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:43,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +30: [2023-05-13 09:38:43,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +23: [2023-05-13 09:38:43,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,030] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 18 + 9: [2023-05-13 09:38:43,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:43,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:43,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +20: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:43,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:43,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:43,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:43,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +30: [2023-05-13 09:38:43,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:43,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:43,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:43,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:43,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:38:43,038] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 252 +20: [2023-05-13 09:38:43,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:43,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:43,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:43,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:43,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:43,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:43,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +27: [2023-05-13 09:38:43,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +30: [2023-05-13 09:38:43,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:43,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +30: [2023-05-13 09:38:43,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +30: [2023-05-13 09:38:43,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:43,044] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 252 + 2: [2023-05-13 09:38:43,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:38:43,045] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 20 +30: [2023-05-13 09:38:43,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:43,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:43,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:43,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:43,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,051] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 20 +31: [2023-05-13 09:38:43,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:43,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt. +19: [2023-05-13 09:38:43,054] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 158 +31: [2023-05-13 09:38:43,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:43,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +31: [2023-05-13 09:38:43,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +20: [2023-05-13 09:38:43,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt. +20: [2023-05-13 09:38:43,058] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 160 +20: [2023-05-13 09:38:43,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt. +20: [2023-05-13 09:38:43,058] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 162 + 8: [2023-05-13 09:38:43,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +19: [2023-05-13 09:38:43,060] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 158 + 8: [2023-05-13 09:38:43,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +28: [2023-05-13 09:38:43,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:38:43,063] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 226 +31: [2023-05-13 09:38:43,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt. +20: [2023-05-13 09:38:43,066] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 164 +20: [2023-05-13 09:38:43,066] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 162 +31: [2023-05-13 09:38:43,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +20: [2023-05-13 09:38:43,067] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 160 + 9: [2023-05-13 09:38:43,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:38:43,070] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 226 + 6: [2023-05-13 09:38:43,070] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 54 + 9: [2023-05-13 09:38:43,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +20: [2023-05-13 09:38:43,074] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 164 + 8: [2023-05-13 09:38:43,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:43,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +18: [2023-05-13 09:38:43,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,077] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 54 + 8: [2023-05-13 09:38:43,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:43,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. +18: [2023-05-13 09:38:43,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. + 1: [2023-05-13 09:38:43,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,080] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 22 + 9: [2023-05-13 09:38:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,087] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 22 + 9: [2023-05-13 09:38:43,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. + 9: [2023-05-13 09:38:43,088] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 72 +18: [2023-05-13 09:38:43,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:43,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:43,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:43,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,096] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 72 + 6: [2023-05-13 09:38:43,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +18: [2023-05-13 09:38:43,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:43,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:43,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:38:43,125] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 250 + 5: [2023-05-13 09:38:43,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:43,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:38:43,126] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 234 + 5: [2023-05-13 09:38:43,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:43,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +31: [2023-05-13 09:38:43,132] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 250 + 6: [2023-05-13 09:38:43,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +18: [2023-05-13 09:38:43,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:43,134] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 234 +18: [2023-05-13 09:38:43,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +18: [2023-05-13 09:38:43,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:43,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt... +22: [2023-05-13 09:38:43,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt. +22: [2023-05-13 09:38:43,148] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 180 +29: [2023-05-13 09:38:43,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:38:43,149] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 238 +22: [2023-05-13 09:38:43,154] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 180 +29: [2023-05-13 09:38:43,158] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 238 + 6: [2023-05-13 09:38:43,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. + 6: [2023-05-13 09:38:43,161] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 50 + 9: [2023-05-13 09:38:43,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +10: [2023-05-13 09:38:43,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. + 9: [2023-05-13 09:38:43,164] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 76 +10: [2023-05-13 09:38:43,164] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 84 + 6: [2023-05-13 09:38:43,169] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 50 +10: [2023-05-13 09:38:43,171] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 84 + 9: [2023-05-13 09:38:43,172] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 76 + 4: [2023-05-13 09:38:43,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:43,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:43,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:43,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:38:43,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. + 3: [2023-05-13 09:38:43,186] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 30 +10: [2023-05-13 09:38:43,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:38:43,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:43,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:38:43,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +14: [2023-05-13 09:38:43,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +14: [2023-05-13 09:38:43,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +14: [2023-05-13 09:38:43,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +14: [2023-05-13 09:38:43,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:43,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:43,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:43,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:43,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,193] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 30 +24: [2023-05-13 09:38:43,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:43,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. + 6: [2023-05-13 09:38:43,197] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 52 +31: [2023-05-13 09:38:43,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:38:43,198] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 254 +27: [2023-05-13 09:38:43,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:38:43,201] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 218 + 2: [2023-05-13 09:38:43,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. + 0: [2023-05-13 09:38:43,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,204] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 24 +14: [2023-05-13 09:38:43,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:43,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:43,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:43,205] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 254 + 2: [2023-05-13 09:38:43,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,206] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 52 + 2: [2023-05-13 09:38:43,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:43,207] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 218 +16: [2023-05-13 09:38:43,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +16: [2023-05-13 09:38:43,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:43,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:43,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +16: [2023-05-13 09:38:43,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:43,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:43,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:43,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,213] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 24 +24: [2023-05-13 09:38:43,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:43,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:43,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:43,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:38:43,218] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 248 + 2: [2023-05-13 09:38:43,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +31: [2023-05-13 09:38:43,224] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 248 +16: [2023-05-13 09:38:43,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:43,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:38:43,225] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 16 +16: [2023-05-13 09:38:43,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:43,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +10: [2023-05-13 09:38:43,226] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 80 +16: [2023-05-13 09:38:43,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:43,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:43,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:43,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:43,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:43,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:43,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:43,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:43,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:38:43,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,232] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 16 +10: [2023-05-13 09:38:43,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:38:43,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +24: [2023-05-13 09:38:43,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:43,235] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 82 + 0: [2023-05-13 09:38:43,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:43,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:43,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:43,236] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 80 + 0: [2023-05-13 09:38:43,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:43,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:43,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +28: [2023-05-13 09:38:43,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +28: [2023-05-13 09:38:43,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +28: [2023-05-13 09:38:43,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +10: [2023-05-13 09:38:43,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +10: [2023-05-13 09:38:43,238] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 86 +10: [2023-05-13 09:38:43,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:43,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:43,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:43,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:38:43,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:38:43,242] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 82 + 0: [2023-05-13 09:38:43,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:43,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:43,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:43,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +25: [2023-05-13 09:38:43,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:38:43,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:43,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:38:43,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:43,245] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 86 +25: [2023-05-13 09:38:43,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:43,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:43,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +15: [2023-05-13 09:38:43,246] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 124 + 2: [2023-05-13 09:38:43,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:43,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:43,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:43,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:43,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:43,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:43,253] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 124 +27: [2023-05-13 09:38:43,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:38:43,253] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 216 +14: [2023-05-13 09:38:43,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:43,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:43,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:38:43,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:38:43,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:38:43,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +16: [2023-05-13 09:38:43,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:43,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:43,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt. +22: [2023-05-13 09:38:43,256] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 176 +28: [2023-05-13 09:38:43,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:38:43,258] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 228 +11: [2023-05-13 09:38:43,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:43,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +16: [2023-05-13 09:38:43,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:43,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +16: [2023-05-13 09:38:43,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +16: [2023-05-13 09:38:43,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:43,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +27: [2023-05-13 09:38:43,259] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 216 +25: [2023-05-13 09:38:43,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:43,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:43,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +16: [2023-05-13 09:38:43,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:43,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:43,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt. +25: [2023-05-13 09:38:43,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:43,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:43,261] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 130 +16: [2023-05-13 09:38:43,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. + 1: [2023-05-13 09:38:43,262] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 14 +16: [2023-05-13 09:38:43,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +19: [2023-05-13 09:38:43,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt. +19: [2023-05-13 09:38:43,262] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 154 +24: [2023-05-13 09:38:43,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:43,263] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 176 +28: [2023-05-13 09:38:43,265] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 228 +16: [2023-05-13 09:38:43,267] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 130 + 1: [2023-05-13 09:38:43,268] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 14 +24: [2023-05-13 09:38:43,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:38:43,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:38:43,269] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 222 +19: [2023-05-13 09:38:43,269] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 154 +24: [2023-05-13 09:38:43,268] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 198 +31: [2023-05-13 09:38:43,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:43,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:43,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:43,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:43,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +17: [2023-05-13 09:38:43,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +17: [2023-05-13 09:38:43,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +17: [2023-05-13 09:38:43,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:43,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:43,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:43,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:38:43,274] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 134 +28: [2023-05-13 09:38:43,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:43,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:43,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +19: [2023-05-13 09:38:43,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:43,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:43,275] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 222 +19: [2023-05-13 09:38:43,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +19: [2023-05-13 09:38:43,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:43,276] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 198 +28: [2023-05-13 09:38:43,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +22: [2023-05-13 09:38:43,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:43,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:43,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:43,280] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 134 +22: [2023-05-13 09:38:43,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. +22: [2023-05-13 09:38:43,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:43,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:43,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,283] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 70 +22: [2023-05-13 09:38:43,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. + 1: [2023-05-13 09:38:43,285] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 12 +17: [2023-05-13 09:38:43,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:43,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt. +22: [2023-05-13 09:38:43,287] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 178 +28: [2023-05-13 09:38:43,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:43,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,290] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 70 +17: [2023-05-13 09:38:43,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:43,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:43,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:43,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:43,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:43,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:43,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:43,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:43,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:43,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:43,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,294] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 12 +22: [2023-05-13 09:38:43,294] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 178 +19: [2023-05-13 09:38:43,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:43,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:38:43,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:38:43,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:38:43,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:43,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:43,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:38:43,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +31: [2023-05-13 09:38:43,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:43,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +26: [2023-05-13 09:38:43,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:43,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:38:43,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:43,305] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 92 +26: [2023-05-13 09:38:43,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:38:43,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +31: [2023-05-13 09:38:43,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:43,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. + 5: [2023-05-13 09:38:43,308] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 42 + 1: [2023-05-13 09:38:43,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:38:43,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,308] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 10 +26: [2023-05-13 09:38:43,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +31: [2023-05-13 09:38:43,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +31: [2023-05-13 09:38:43,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:38:43,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +23: [2023-05-13 09:38:43,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +23: [2023-05-13 09:38:43,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:43,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:43,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +23: [2023-05-13 09:38:43,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:43,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:43,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:43,311] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 92 +20: [2023-05-13 09:38:43,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:43,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:43,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:43,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:43,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:43,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:43,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:43,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +19: [2023-05-13 09:38:43,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:43,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +15: [2023-05-13 09:38:43,313] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 122 +13: [2023-05-13 09:38:43,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:43,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,314] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 42 +11: [2023-05-13 09:38:43,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +11: [2023-05-13 09:38:43,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +19: [2023-05-13 09:38:43,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +11: [2023-05-13 09:38:43,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:43,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,315] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 10 +17: [2023-05-13 09:38:43,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:38:43,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +11: [2023-05-13 09:38:43,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +26: [2023-05-13 09:38:43,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:43,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +15: [2023-05-13 09:38:43,321] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 122 +26: [2023-05-13 09:38:43,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:43,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. + 6: [2023-05-13 09:38:43,325] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 48 +23: [2023-05-13 09:38:43,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:43,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:43,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:43,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:43,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:43,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +20: [2023-05-13 09:38:43,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:43,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:43,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:43,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:43,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:43,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:43,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:43,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:43,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:43,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:43,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,332] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 48 +29: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:43,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:43,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:43,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:43,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:43,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +17: [2023-05-13 09:38:43,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +19: [2023-05-13 09:38:43,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. + 3: [2023-05-13 09:38:43,336] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 26 +15: [2023-05-13 09:38:43,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +19: [2023-05-13 09:38:43,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +15: [2023-05-13 09:38:43,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:43,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:43,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:43,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +13: [2023-05-13 09:38:43,343] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 108 + 3: [2023-05-13 09:38:43,344] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 26 +19: [2023-05-13 09:38:43,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt. +19: [2023-05-13 09:38:43,346] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 156 + 3: [2023-05-13 09:38:43,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +13: [2023-05-13 09:38:43,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:43,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +13: [2023-05-13 09:38:43,350] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 108 + 1: [2023-05-13 09:38:43,350] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 8 + 3: [2023-05-13 09:38:43,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:43,355] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 156 + 1: [2023-05-13 09:38:43,356] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 8 +20: [2023-05-13 09:38:43,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:43,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:43,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:43,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:43,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:43,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:43,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:43,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:43,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:43,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +26: [2023-05-13 09:38:43,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:38:43,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:43,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt. + 9: [2023-05-13 09:38:43,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +21: [2023-05-13 09:38:43,362] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 174 +23: [2023-05-13 09:38:43,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +26: [2023-05-13 09:38:43,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +26: [2023-05-13 09:38:43,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:43,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +26: [2023-05-13 09:38:43,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +13: [2023-05-13 09:38:43,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:43,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt. +15: [2023-05-13 09:38:43,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:43,364] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 186 +26: [2023-05-13 09:38:43,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:38:43,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:43,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:43,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:43,368] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 174 +26: [2023-05-13 09:38:43,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +20: [2023-05-13 09:38:43,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +23: [2023-05-13 09:38:43,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:43,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:43,371] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 186 +23: [2023-05-13 09:38:43,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +29: [2023-05-13 09:38:43,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:43,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +27: [2023-05-13 09:38:43,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:43,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +27: [2023-05-13 09:38:43,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:43,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:43,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:43,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:43,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:43,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:43,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:43,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +29: [2023-05-13 09:38:43,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +29: [2023-05-13 09:38:43,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +29: [2023-05-13 09:38:43,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +29: [2023-05-13 09:38:43,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +13: [2023-05-13 09:38:43,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +23: [2023-05-13 09:38:43,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:43,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt. +24: [2023-05-13 09:38:43,382] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 192 +12: [2023-05-13 09:38:43,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +12: [2023-05-13 09:38:43,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +12: [2023-05-13 09:38:43,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +12: [2023-05-13 09:38:43,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:43,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. + 0: [2023-05-13 09:38:43,387] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 0 +27: [2023-05-13 09:38:43,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:43,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +27: [2023-05-13 09:38:43,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:43,390] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 192 +27: [2023-05-13 09:38:43,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:43,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,394] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 0 + 6: [2023-05-13 09:38:43,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:43,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +21: [2023-05-13 09:38:43,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +21: [2023-05-13 09:38:43,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +21: [2023-05-13 09:38:43,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +21: [2023-05-13 09:38:43,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +12: [2023-05-13 09:38:43,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:43,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:43,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:43,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:43,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:43,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt. +24: [2023-05-13 09:38:43,399] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 196 + 0: could not find arguments in the checkpoint ... + 0: checkpoint version 3.0 +15: [2023-05-13 09:38:43,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:43,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:38:43,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:38:43,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +30: [2023-05-13 09:38:43,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:43,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:43,406] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 196 +18: [2023-05-13 09:38:43,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +18: [2023-05-13 09:38:43,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:43,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +11: [2023-05-13 09:38:43,407] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 94 +30: [2023-05-13 09:38:43,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:43,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:38:43,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:43,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:43,407] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 88 +18: [2023-05-13 09:38:43,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:43,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt. +30: [2023-05-13 09:38:43,409] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 246 + 5: [2023-05-13 09:38:43,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +27: [2023-05-13 09:38:43,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:43,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:43,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +21: [2023-05-13 09:38:43,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:43,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:43,415] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 94 + 6: [2023-05-13 09:38:43,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:43,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +11: [2023-05-13 09:38:43,417] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 88 +30: [2023-05-13 09:38:43,417] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 246 +18: [2023-05-13 09:38:43,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:43,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:43,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:43,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:43,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:43,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:43,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:43,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:43,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:43,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:43,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:38:43,426] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 150 + 6: [2023-05-13 09:38:43,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +27: [2023-05-13 09:38:43,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:43,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt. + 6: [2023-05-13 09:38:43,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:43,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:43,430] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 170 +27: [2023-05-13 09:38:43,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:43,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:43,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:43,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:43,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:43,433] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 150 +27: [2023-05-13 09:38:43,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:43,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:43,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:43,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +21: [2023-05-13 09:38:43,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:43,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:38:43,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +14: [2023-05-13 09:38:43,437] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 112 +21: [2023-05-13 09:38:43,438] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 170 +21: [2023-05-13 09:38:43,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:43,444] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 112 + 2: [2023-05-13 09:38:43,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:43,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +21: [2023-05-13 09:38:43,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:43,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +21: [2023-05-13 09:38:43,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:43,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +23: [2023-05-13 09:38:43,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt. +23: [2023-05-13 09:38:43,451] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 184 + 1: [2023-05-13 09:38:43,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:43,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:43,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:43,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:43,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:43,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:43,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:38:43,456] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 220 +30: [2023-05-13 09:38:43,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:43,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +21: [2023-05-13 09:38:43,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:43,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +23: [2023-05-13 09:38:43,459] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 184 +30: [2023-05-13 09:38:43,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +16: [2023-05-13 09:38:43,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:38:43,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:43,460] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 132 +10: [2023-05-13 09:38:43,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:43,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:43,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +10: [2023-05-13 09:38:43,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +10: [2023-05-13 09:38:43,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +27: [2023-05-13 09:38:43,463] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 220 +30: [2023-05-13 09:38:43,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:43,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:43,467] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 132 +24: [2023-05-13 09:38:43,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:38:43,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:38:43,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +21: [2023-05-13 09:38:43,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:43,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. + 9: [2023-05-13 09:38:43,469] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 74 + 1: [2023-05-13 09:38:43,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,476] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 74 +10: [2023-05-13 09:38:43,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:43,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:43,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:38:43,477] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 212 +10: [2023-05-13 09:38:43,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:43,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:43,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt. +30: [2023-05-13 09:38:43,482] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 244 + 4: [2023-05-13 09:38:43,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:43,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:43,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +13: [2023-05-13 09:38:43,483] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 104 +26: [2023-05-13 09:38:43,484] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 212 + 4: [2023-05-13 09:38:43,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:38:43,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +24: [2023-05-13 09:38:43,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:43,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:43,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:43,487] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 116 +30: [2023-05-13 09:38:43,489] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 244 +13: [2023-05-13 09:38:43,490] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 104 +14: [2023-05-13 09:38:43,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +14: [2023-05-13 09:38:43,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +14: [2023-05-13 09:38:43,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +14: [2023-05-13 09:38:43,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:43,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +15: [2023-05-13 09:38:43,492] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 120 + 1: [2023-05-13 09:38:43,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:43,493] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 116 + 4: [2023-05-13 09:38:43,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:38:43,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt. +23: [2023-05-13 09:38:43,496] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 190 + 4: [2023-05-13 09:38:43,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:43,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt. +22: [2023-05-13 09:38:43,498] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 182 + 4: [2023-05-13 09:38:43,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. + 8: [2023-05-13 09:38:43,500] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 68 + 2: [2023-05-13 09:38:43,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +23: [2023-05-13 09:38:43,502] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 190 + 2: [2023-05-13 09:38:43,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:43,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:43,503] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 120 + 4: [2023-05-13 09:38:43,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. + 4: [2023-05-13 09:38:43,504] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 38 + 2: [2023-05-13 09:38:43,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +10: [2023-05-13 09:38:43,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:43,505] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 182 + 2: [2023-05-13 09:38:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,507] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 68 + 1: [2023-05-13 09:38:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:38:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +22: [2023-05-13 09:38:43,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:38:43,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:38:43,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:38:43,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,511] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 38 +10: [2023-05-13 09:38:43,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:43,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +16: [2023-05-13 09:38:43,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +16: [2023-05-13 09:38:43,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +16: [2023-05-13 09:38:43,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +10: [2023-05-13 09:38:43,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:43,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:38:43,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:38:43,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:38:43,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +10: [2023-05-13 09:38:43,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:43,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +24: [2023-05-13 09:38:43,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:43,516] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 106 +10: [2023-05-13 09:38:43,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +24: [2023-05-13 09:38:43,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +26: [2023-05-13 09:38:43,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:38:43,519] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 214 +17: [2023-05-13 09:38:43,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +17: [2023-05-13 09:38:43,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:38:43,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:43,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:43,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +17: [2023-05-13 09:38:43,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:38:43,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +10: [2023-05-13 09:38:43,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +24: [2023-05-13 09:38:43,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. + 4: [2023-05-13 09:38:43,523] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 36 +13: [2023-05-13 09:38:43,523] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 106 +26: [2023-05-13 09:38:43,526] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 214 +22: [2023-05-13 09:38:43,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:43,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:43,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:43,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:43,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:43,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:43,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:43,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:43,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:43,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,530] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 36 +11: [2023-05-13 09:38:43,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:43,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:43,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:43,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:43,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:43,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:43,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:43,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:43,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:43,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:38:43,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:38:43,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:38:43,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:43,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt. +30: [2023-05-13 09:38:43,545] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 242 + 8: [2023-05-13 09:38:43,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. +14: [2023-05-13 09:38:43,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,547] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 66 + 3: [2023-05-13 09:38:43,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. + 3: [2023-05-13 09:38:43,547] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 28 + 5: [2023-05-13 09:38:43,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. + 5: [2023-05-13 09:38:43,548] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 46 + 0: [2023-05-13 09:38:43,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. + 0: [2023-05-13 09:38:43,551] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 4 +30: [2023-05-13 09:38:43,551] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 242 + 8: [2023-05-13 09:38:43,553] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 66 + 3: [2023-05-13 09:38:43,553] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 28 + 5: [2023-05-13 09:38:43,554] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 46 +30: [2023-05-13 09:38:43,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:38:43,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +30: [2023-05-13 09:38:43,556] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 240 +12: [2023-05-13 09:38:43,556] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 96 + 0: [2023-05-13 09:38:43,557] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 4 +13: [2023-05-13 09:38:43,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:38:43,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:43,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt. +23: [2023-05-13 09:38:43,558] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 188 +22: [2023-05-13 09:38:43,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +16: [2023-05-13 09:38:43,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:43,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:43,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +16: [2023-05-13 09:38:43,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +16: [2023-05-13 09:38:43,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:43,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:43,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:43,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:43,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:38:43,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:43,562] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 110 +22: [2023-05-13 09:38:43,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +30: [2023-05-13 09:38:43,563] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 240 +12: [2023-05-13 09:38:43,564] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 96 +16: [2023-05-13 09:38:43,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:43,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:43,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:43,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:43,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:38:43,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +16: [2023-05-13 09:38:43,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:43,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:43,565] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 188 +11: [2023-05-13 09:38:43,565] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 90 +22: [2023-05-13 09:38:43,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:43,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +13: [2023-05-13 09:38:43,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:38:43,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +22: [2023-05-13 09:38:43,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:43,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. +13: [2023-05-13 09:38:43,569] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 110 + 7: [2023-05-13 09:38:43,569] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 60 +11: [2023-05-13 09:38:43,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:43,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:43,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. + 5: [2023-05-13 09:38:43,572] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 44 +11: [2023-05-13 09:38:43,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:43,574] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 90 + 7: [2023-05-13 09:38:43,576] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 60 +17: [2023-05-13 09:38:43,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,578] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 44 +17: [2023-05-13 09:38:43,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:43,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:43,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +20: [2023-05-13 09:38:43,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +20: [2023-05-13 09:38:43,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +20: [2023-05-13 09:38:43,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +13: [2023-05-13 09:38:43,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:43,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:43,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:43,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:43,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:43,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:43,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:38:43,587] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 210 +17: [2023-05-13 09:38:43,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:38:43,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. + 7: [2023-05-13 09:38:43,593] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 56 +26: [2023-05-13 09:38:43,594] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 210 +20: [2023-05-13 09:38:43,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:43,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:43,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:43,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:38:43,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:38:43,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:38:43,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,599] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 56 +26: [2023-05-13 09:38:43,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:38:43,602] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 208 +13: [2023-05-13 09:38:43,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:43,608] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 208 +13: [2023-05-13 09:38:43,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:43,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:43,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:38:43,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:43,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:43,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:43,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:43,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:43,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:43,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. + 5: [2023-05-13 09:38:43,617] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 40 +13: [2023-05-13 09:38:43,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. + 0: [2023-05-13 09:38:43,619] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 6 +20: [2023-05-13 09:38:43,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:43,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:43,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,624] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 40 +20: [2023-05-13 09:38:43,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:38:43,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:38:43,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,625] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 6 +25: [2023-05-13 09:38:43,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:38:43,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +20: [2023-05-13 09:38:43,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:43,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:43,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:38:43,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:38:43,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:38:43,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +20: [2023-05-13 09:38:43,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +31: [2023-05-13 09:38:43,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:38:43,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +12: [2023-05-13 09:38:43,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +20: [2023-05-13 09:38:43,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +12: [2023-05-13 09:38:43,632] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 102 +31: [2023-05-13 09:38:43,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:38:43,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:38:43,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:43,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:38:43,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:43,639] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 102 +25: [2023-05-13 09:38:43,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:43,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:43,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:38:43,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:38:43,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:43,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:38:43,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:43,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:43,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +29: [2023-05-13 09:38:43,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +29: [2023-05-13 09:38:43,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:38:43,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:43,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:43,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:38:43,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:43,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,649] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 2 +28: [2023-05-13 09:38:43,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:38:43,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:43,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:43,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:43,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:38:43,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,659] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 2 + 8: [2023-05-13 09:38:43,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:43,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:43,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:43,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:43,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:43,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:43,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:43,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:38:43,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:38:43,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +23: [2023-05-13 09:38:43,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:43,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:43,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +31: [2023-05-13 09:38:43,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +23: [2023-05-13 09:38:43,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:43,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:43,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:43,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:38:43,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:43,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +23: [2023-05-13 09:38:43,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:43,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt. +25: [2023-05-13 09:38:43,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:38:43,681] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 140 +31: [2023-05-13 09:38:43,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:43,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt. +17: [2023-05-13 09:38:43,683] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 136 +31: [2023-05-13 09:38:43,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:43,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +26: [2023-05-13 09:38:43,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:43,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +26: [2023-05-13 09:38:43,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +26: [2023-05-13 09:38:43,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +18: [2023-05-13 09:38:43,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:38:43,689] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 146 + 8: [2023-05-13 09:38:43,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +23: [2023-05-13 09:38:43,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:43,689] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 140 + 2: [2023-05-13 09:38:43,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:38:43,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +17: [2023-05-13 09:38:43,690] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 136 + 2: [2023-05-13 09:38:43,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:38:43,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:38:43,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:38:43,691] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 204 + 7: [2023-05-13 09:38:43,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:43,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:43,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:38:43,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. + 7: [2023-05-13 09:38:43,694] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 62 + 4: [2023-05-13 09:38:43,694] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 34 + 7: [2023-05-13 09:38:43,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:38:43,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +10: [2023-05-13 09:38:43,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:38:43,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:38:43,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:38:43,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +18: [2023-05-13 09:38:43,695] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 146 + 8: [2023-05-13 09:38:43,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:38:43,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:38:43,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:43,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:38:43,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:38:43,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:43,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +19: [2023-05-13 09:38:43,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:38:43,700] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 204 +21: [2023-05-13 09:38:43,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +19: [2023-05-13 09:38:43,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +19: [2023-05-13 09:38:43,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:38:43,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:38:43,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:38:43,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:38:43,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +29: [2023-05-13 09:38:43,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +21: [2023-05-13 09:38:43,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 7: [2023-05-13 09:38:43,702] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 62 +26: [2023-05-13 09:38:43,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:43,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:43,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:43,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:43,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:43,702] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 34 +15: [2023-05-13 09:38:43,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +21: [2023-05-13 09:38:43,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +26: [2023-05-13 09:38:43,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:43,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +26: [2023-05-13 09:38:43,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:38:43,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:43,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt. +21: [2023-05-13 09:38:43,707] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 172 +10: [2023-05-13 09:38:43,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:38:43,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,708] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 58 +29: [2023-05-13 09:38:43,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +27: [2023-05-13 09:38:43,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:43,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:43,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:43,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:43,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +10: [2023-05-13 09:38:43,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:43,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:43,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +27: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 9: [2023-05-13 09:38:43,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +19: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:43,714] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 58 + 3: [2023-05-13 09:38:43,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:43,715] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 172 + 3: [2023-05-13 09:38:43,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:43,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:43,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:38:43,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:43,716] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 202 +12: [2023-05-13 09:38:43,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:43,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:43,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:43,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +14: [2023-05-13 09:38:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:43,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:43,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:43,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:38:43,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:43,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +15: [2023-05-13 09:38:43,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:43,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:43,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:43,722] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 202 +21: [2023-05-13 09:38:43,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:43,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +22: [2023-05-13 09:38:43,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +26: [2023-05-13 09:38:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +15: [2023-05-13 09:38:43,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:43,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +26: [2023-05-13 09:38:43,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:43,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt. +10: [2023-05-13 09:38:43,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:43,727] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 138 +10: [2023-05-13 09:38:43,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:43,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:43,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:43,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:43,733] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 138 +14: [2023-05-13 09:38:43,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:43,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +26: [2023-05-13 09:38:43,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:43,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:43,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +26: [2023-05-13 09:38:43,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +12: [2023-05-13 09:38:43,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:38:43,737] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 98 +14: [2023-05-13 09:38:43,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:43,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:43,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:43,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:43,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:43,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:43,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:38:43,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:43,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 5: [2023-05-13 09:38:43,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:38:43,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +16: [2023-05-13 09:38:43,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 5: [2023-05-13 09:38:43,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:38:43,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:38:43,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:43,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +12: [2023-05-13 09:38:43,744] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 98 +27: [2023-05-13 09:38:43,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:43,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. + 3: [2023-05-13 09:38:43,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:43,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:43,746] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 114 + 2: [2023-05-13 09:38:43,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +24: [2023-05-13 09:38:43,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:38:43,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:38:43,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:38:43,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:38:43,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:43,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:43,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:43,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +10: [2023-05-13 09:38:43,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:38:43,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +19: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:43,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 4: [2023-05-13 09:38:43,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:38:43,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:43,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:43,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:43,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:43,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +21: [2023-05-13 09:38:43,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:43,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:38:43,754] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 114 +12: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +12: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +18: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 3: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +21: [2023-05-13 09:38:43,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +19: [2023-05-13 09:38:43,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +19: [2023-05-13 09:38:43,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +19: [2023-05-13 09:38:43,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:38:43,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +12: [2023-05-13 09:38:43,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:38:43,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +19: [2023-05-13 09:38:43,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:43,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:38:43,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:43,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:38:43,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +21: [2023-05-13 09:38:43,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:43,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:43,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:43,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +24: [2023-05-13 09:38:43,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:43,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:43,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:43,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:43,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:38:43,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:43,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:43,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +24: [2023-05-13 09:38:43,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt. +24: [2023-05-13 09:38:43,762] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 194 +21: [2023-05-13 09:38:43,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +22: [2023-05-13 09:38:43,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:43,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:43,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:43,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:38:43,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +24: [2023-05-13 09:38:43,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:43,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:43,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:43,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:43,769] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 194 + 9: [2023-05-13 09:38:43,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:38:43,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:43,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:43,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:43,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:38:43,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:43,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:43,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:43,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:43,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:43,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +14: [2023-05-13 09:38:43,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:43,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:38:43,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +22: [2023-05-13 09:38:43,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +14: [2023-05-13 09:38:43,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +22: [2023-05-13 09:38:43,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. + 8: [2023-05-13 09:38:43,782] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 64 +30: [2023-05-13 09:38:43,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:43,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:43,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:38:43,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:38:43,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:43,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 8: [2023-05-13 09:38:43,789] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 64 +21: [2023-05-13 09:38:43,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt. +20: [2023-05-13 09:38:43,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +20: [2023-05-13 09:38:43,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:38:43,792] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 168 +20: [2023-05-13 09:38:43,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +16: [2023-05-13 09:38:43,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:43,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:43,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:38:43,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:38:43,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:38:43,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:43,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:43,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +17: [2023-05-13 09:38:43,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +17: [2023-05-13 09:38:43,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +17: [2023-05-13 09:38:43,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:38:43,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:38:43,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:38:43,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +13: [2023-05-13 09:38:43,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +16: [2023-05-13 09:38:43,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:43,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:43,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:43,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:38:43,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:38:43,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:38:43,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:38:43,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:43,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:43,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:43,799] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 168 +16: [2023-05-13 09:38:43,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:38:43,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:38:43,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:43,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:38:43,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:43,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:43,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:43,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +24: [2023-05-13 09:38:43,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:43,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt. +17: [2023-05-13 09:38:43,803] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 142 +18: [2023-05-13 09:38:43,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:43,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:43,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt. +25: [2023-05-13 09:38:43,807] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 200 + 1: [2023-05-13 09:38:43,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:43,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:43,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:43,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:43,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:38:43,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:43,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:43,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:43,811] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 142 +17: [2023-05-13 09:38:43,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:43,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:43,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:43,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:43,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:43,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:43,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:38:43,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:38:43,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:43,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:38:43,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:43,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:43,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:43,816] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 200 +30: [2023-05-13 09:38:43,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:43,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +30: [2023-05-13 09:38:43,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:43,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +30: [2023-05-13 09:38:43,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:43,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:43,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:43,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:38:43,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +26: [2023-05-13 09:38:43,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:38:43,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:43,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:38:43,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:38:43,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:38:43,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:38:43,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:38:43,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:43,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:43,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:43,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:43,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:43,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:38:43,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:38:43,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:43,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:38:43,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:43,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +20: [2023-05-13 09:38:43,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +20: [2023-05-13 09:38:43,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:38:43,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:43,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:43,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +29: [2023-05-13 09:38:43,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +29: [2023-05-13 09:38:43,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +29: [2023-05-13 09:38:43,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +17: [2023-05-13 09:38:43,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:38:43,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:38:43,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:43,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:43,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:43,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:43,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:38:43,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:43,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:38:43,851] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 144 +20: [2023-05-13 09:38:43,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:38:43,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +13: [2023-05-13 09:38:43,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:38:43,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +20: [2023-05-13 09:38:43,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:43,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +20: [2023-05-13 09:38:43,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +23: [2023-05-13 09:38:43,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:43,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:43,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:43,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:43,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:43,858] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 144 +26: [2023-05-13 09:38:43,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:43,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:43,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:38:43,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:43,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:43,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:43,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:43,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:43,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:43,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:38:43,872] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 100 +23: [2023-05-13 09:38:43,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:43,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:43,880] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 100 +14: [2023-05-13 09:38:43,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +14: [2023-05-13 09:38:43,882] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 118 +18: [2023-05-13 09:38:43,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:38:43,882] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 148 + 6: [2023-05-13 09:38:43,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:38:43,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:38:43,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:43,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +14: [2023-05-13 09:38:43,891] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 118 +26: [2023-05-13 09:38:43,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:43,891] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 148 +12: [2023-05-13 09:38:43,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +12: [2023-05-13 09:38:43,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:38:43,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:43,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +12: [2023-05-13 09:38:43,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +26: [2023-05-13 09:38:43,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +23: [2023-05-13 09:38:43,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:38:43,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:43,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:43,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:38:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:38:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:43,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +23: [2023-05-13 09:38:43,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:43,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:43,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:43,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:43,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:43,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:38:43,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:38:43,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:43,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:43,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:43,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:43,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:43,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:38:43,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:43,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:38:43,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +18: [2023-05-13 09:38:43,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +18: [2023-05-13 09:38:43,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +18: [2023-05-13 09:38:43,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 6: [2023-05-13 09:38:43,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:43,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:38:43,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:43,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:43,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:43,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:43,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:43,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:43,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:43,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:43,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:43,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:43,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:38:43,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:38:43,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:38:43,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:38:43,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:38:43,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +12: [2023-05-13 09:38:43,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:38:43,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:38:43,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:43,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:38:43,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 3: [2023-05-13 09:38:43,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:38:43,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:38:43,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 2: [2023-05-13 09:38:43,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:38:43,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +28: [2023-05-13 09:38:43,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:38:43,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:38:43,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +28: [2023-05-13 09:38:43,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:38:43,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:43,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:43,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +28: [2023-05-13 09:38:43,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:38:43,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. + 1: [2023-05-13 09:38:43,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +18: [2023-05-13 09:38:43,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:43,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:38:43,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:43,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:38:43,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:38:43,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:38:43,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:38:43,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:38:43,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:43,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:38:43,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:43,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:43,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:43,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:43,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 2: [2023-05-13 09:38:43,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 1: [2023-05-13 09:38:43,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:43,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +28: [2023-05-13 09:38:43,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +28: [2023-05-13 09:38:43,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +31: [2023-05-13 09:38:43,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:38:43,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +28: [2023-05-13 09:38:43,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 2: [2023-05-13 09:38:43,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +10: [2023-05-13 09:38:43,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 2: [2023-05-13 09:38:43,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +31: [2023-05-13 09:38:43,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:43,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +31: [2023-05-13 09:38:43,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:43,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:38:43,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:38:43,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:38:43,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:38:43,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:38:43,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:38:43,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:43,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:43,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +31: [2023-05-13 09:38:43,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:43,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +19: [2023-05-13 09:38:43,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:38:43,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +10: [2023-05-13 09:38:43,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +31: [2023-05-13 09:38:43,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +31: [2023-05-13 09:38:43,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +31: [2023-05-13 09:38:43,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:38:43,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:38:43,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:38:43,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:38:43,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:38:43,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:38:43,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:38:43,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:38:43,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:38:44,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:44,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:44,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:44,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:38:44,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:44,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:44,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +10: [2023-05-13 09:38:44,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:44,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:44,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:44,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:44,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:44,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:38:44,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:44,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +10: [2023-05-13 09:38:44,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:38:44,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:38:44,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:38:44,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +10: [2023-05-13 09:38:44,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +10: [2023-05-13 09:38:44,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +10: [2023-05-13 09:38:44,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:44,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:44,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:44,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:44,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:44,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:44,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:44,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:44,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:44,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:44,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:44,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +25: [2023-05-13 09:38:44,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +25: [2023-05-13 09:38:44,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 2: [2023-05-13 09:38:44,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt... + 2: [2023-05-13 09:38:44,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt... + 2: [2023-05-13 09:38:44,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt... + 2: [2023-05-13 09:38:44,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt... +25: [2023-05-13 09:38:44,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:38:44,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt... +28: [2023-05-13 09:38:44,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt... +28: [2023-05-13 09:38:44,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt... +28: [2023-05-13 09:38:44,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt... +21: [2023-05-13 09:38:44,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:38:44,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +15: [2023-05-13 09:38:44,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +15: [2023-05-13 09:38:44,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:38:44,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:38:44,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_37-model_01-model_states.pt. +19: [2023-05-13 09:38:44,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:44,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:38:44,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:38:44,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:38:44,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt... +19: [2023-05-13 09:38:44,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +21: [2023-05-13 09:38:44,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:44,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +19: [2023-05-13 09:38:44,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:44,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:44,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +19: [2023-05-13 09:38:44,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:38:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:44,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:44,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:44,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:38:44,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +25: [2023-05-13 09:38:44,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:44,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +25: [2023-05-13 09:38:44,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:44,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:44,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:44,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:44,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +25: [2023-05-13 09:38:44,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +25: [2023-05-13 09:38:44,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +25: [2023-05-13 09:38:44,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +25: [2023-05-13 09:38:44,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +25: [2023-05-13 09:38:44,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:38:44,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:44,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:44,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +21: [2023-05-13 09:38:44,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:44,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:44,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:38:44,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:44,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:44,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +21: [2023-05-13 09:38:44,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:38:44,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt... +10: [2023-05-13 09:38:44,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt... +10: [2023-05-13 09:38:44,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt... +10: [2023-05-13 09:38:44,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt... +30: [2023-05-13 09:38:44,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:44,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:38:44,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:44,045] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 206 +30: [2023-05-13 09:38:44,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:38:44,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:38:44,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:44,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:44,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:44,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +19: [2023-05-13 09:38:44,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:44,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:38:44,053] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 206 +30: [2023-05-13 09:38:44,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:38:44,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:38:44,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:44,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:38:44,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:44,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt... +25: [2023-05-13 09:38:44,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt... +25: [2023-05-13 09:38:44,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt... +25: [2023-05-13 09:38:44,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt... + 4: [2023-05-13 09:38:44,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 4: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 4: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 4: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:44,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:44,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 4: [2023-05-13 09:38:44,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:44,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:38:44,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:38:44,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:38:44,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:38:44,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:44,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:38:44,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:38:44,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:38:44,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:38:44,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:44,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:38:44,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt... +22: [2023-05-13 09:38:44,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:38:44,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:38:44,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:38:44,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:38:44,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:38:44,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:38:44,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:38:44,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +14: [2023-05-13 09:38:44,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +14: [2023-05-13 09:38:44,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +14: [2023-05-13 09:38:44,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +14: [2023-05-13 09:38:44,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt... + 4: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt... + 4: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt... + 4: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt... +22: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:44,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:44,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:44,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:44,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:44,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:38:44,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:38:44,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:38:44,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:38:44,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:38:44,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:44,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +24: [2023-05-13 09:38:44,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:44,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:44,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:38:44,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:38:44,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +24: [2023-05-13 09:38:44,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +24: [2023-05-13 09:38:44,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:38:44,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +14: [2023-05-13 09:38:44,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:44,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +24: [2023-05-13 09:38:44,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +24: [2023-05-13 09:38:44,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:38:44,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +14: [2023-05-13 09:38:44,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +14: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +24: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +14: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +26: [2023-05-13 09:38:44,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +26: [2023-05-13 09:38:44,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +13: [2023-05-13 09:38:44,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +13: [2023-05-13 09:38:44,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +26: [2023-05-13 09:38:44,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +26: [2023-05-13 09:38:44,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +13: [2023-05-13 09:38:44,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +13: [2023-05-13 09:38:44,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +14: [2023-05-13 09:38:44,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:38:44,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +14: [2023-05-13 09:38:44,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +14: [2023-05-13 09:38:44,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +14: [2023-05-13 09:38:44,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:44,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:38:44,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 8: [2023-05-13 09:38:44,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +14: [2023-05-13 09:38:44,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 9: [2023-05-13 09:38:44,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt... + 9: [2023-05-13 09:38:44,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt... + 9: [2023-05-13 09:38:44,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt... + 9: [2023-05-13 09:38:44,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt... +20: [2023-05-13 09:38:44,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +20: [2023-05-13 09:38:44,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +20: [2023-05-13 09:38:44,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:38:44,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:38:44,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:38:44,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:38:44,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:38:44,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +26: [2023-05-13 09:38:44,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:44,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:44,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:44,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:44,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:44,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +26: [2023-05-13 09:38:44,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:44,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:44,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +26: [2023-05-13 09:38:44,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:44,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:38:44,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:38:44,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +26: [2023-05-13 09:38:44,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:44,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:44,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:44,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:38:44,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +26: [2023-05-13 09:38:44,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +26: [2023-05-13 09:38:44,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +26: [2023-05-13 09:38:44,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +26: [2023-05-13 09:38:44,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:38:44,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:38:44,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:38:44,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:38:44,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:44,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:44,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:38:44,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:38:44,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:38:44,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:38:44,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +20: [2023-05-13 09:38:44,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:44,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +20: [2023-05-13 09:38:44,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +20: [2023-05-13 09:38:44,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +20: [2023-05-13 09:38:44,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +20: [2023-05-13 09:38:44,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:44,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +20: [2023-05-13 09:38:44,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +20: [2023-05-13 09:38:44,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +20: [2023-05-13 09:38:44,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +20: [2023-05-13 09:38:44,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:38:44,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt... +22: [2023-05-13 09:38:44,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt... +22: [2023-05-13 09:38:44,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt... +22: [2023-05-13 09:38:44,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt... +14: [2023-05-13 09:38:44,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt... +14: [2023-05-13 09:38:44,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt... +14: [2023-05-13 09:38:44,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt... +14: [2023-05-13 09:38:44,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:38:44,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt... + 8: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt... + 8: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt... + 8: [2023-05-13 09:38:44,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:38:44,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:44,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:44,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:44,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:38:44,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:38:44,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:44,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:38:44,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:44,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:38:44,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:38:44,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:38:44,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:38:44,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 7: [2023-05-13 09:38:44,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt... + 7: [2023-05-13 09:38:44,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt... + 7: [2023-05-13 09:38:44,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt... + 7: [2023-05-13 09:38:44,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:38:44,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:38:44,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:38:44,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:38:44,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:38:44,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:38:44,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:38:44,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:38:44,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:38:44,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt... +20: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt... +20: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 5: [2023-05-13 09:38:44,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 5: [2023-05-13 09:38:44,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 5: [2023-05-13 09:38:44,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:38:44,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 5: [2023-05-13 09:38:44,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:38:44,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:38:44,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:38:44,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:38:44,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt... +11: [2023-05-13 09:38:44,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:38:44,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:38:44,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:38:44,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:38:44,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:44,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:44,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:44,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:44,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +11: [2023-05-13 09:38:44,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:44,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +11: [2023-05-13 09:38:44,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:44,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:44,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:38:44,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +11: [2023-05-13 09:38:44,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +11: [2023-05-13 09:38:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +11: [2023-05-13 09:38:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +11: [2023-05-13 09:38:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +11: [2023-05-13 09:38:44,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:38:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:38:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:38:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:38:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt... +29: [2023-05-13 09:38:44,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +29: [2023-05-13 09:38:44,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +29: [2023-05-13 09:38:44,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +29: [2023-05-13 09:38:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:38:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:38:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:38:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:38:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:38:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:38:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:38:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:38:44,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +27: [2023-05-13 09:38:44,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:38:44,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:44,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:44,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +29: [2023-05-13 09:38:44,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:44,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +15: [2023-05-13 09:38:44,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:38:44,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +17: [2023-05-13 09:38:44,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:44,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +17: [2023-05-13 09:38:44,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:44,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:44,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:38:44,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +17: [2023-05-13 09:38:44,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:38:44,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:38:44,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:38:44,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:38:44,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:44,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:44,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:44,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:38:44,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:44,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +17: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +17: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 6: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 6: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +29: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +12: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +12: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +29: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +29: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +27: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +12: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +29: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +29: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:38:44,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +29: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +29: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +18: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +18: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:38:44,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 1: [2023-05-13 09:38:44,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:38:44,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +18: [2023-05-13 09:38:44,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +18: [2023-05-13 09:38:44,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:44,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:44,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:38:44,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 1: [2023-05-13 09:38:44,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:38:44,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:38:44,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 0: [2023-05-13 09:38:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:38:44,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:38:44,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt... +11: [2023-05-13 09:38:44,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt... +11: [2023-05-13 09:38:44,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt... +11: [2023-05-13 09:38:44,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:38:44,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:44,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:44,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:44,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +27: [2023-05-13 09:38:44,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:44,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:38:44,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:44,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:44,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 6: [2023-05-13 09:38:44,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:44,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:44,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:44,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 6: [2023-05-13 09:38:44,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:44,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 6: [2023-05-13 09:38:44,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:38:44,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +12: [2023-05-13 09:38:44,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:44,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:44,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +27: [2023-05-13 09:38:44,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:44,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:38:44,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:44,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:44,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:44,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +12: [2023-05-13 09:38:44,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +12: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 6: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:44,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:44,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:44,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:44,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:44,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:44,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:44,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:44,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:38:44,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:44,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:44,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:38:44,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:38:44,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +17: [2023-05-13 09:38:44,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:38:44,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:38:44,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:38:44,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:38:44,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:38:44,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:38:44,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:38:44,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:38:44,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:38:44,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:38:44,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:44,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +21: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +21: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +21: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +21: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. + 3: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +29: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt... +29: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt... +29: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt... +29: [2023-05-13 09:38:44,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:38:44,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:38:44,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:38:44,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:38:44,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:38:44,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt... +18: [2023-05-13 09:38:44,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt... +18: [2023-05-13 09:38:44,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt... +18: [2023-05-13 09:38:44,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:38:44,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:38:44,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:38:44,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:38:44,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... +12: [2023-05-13 09:38:44,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +12: [2023-05-13 09:38:44,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +12: [2023-05-13 09:38:44,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:44,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:44,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +12: [2023-05-13 09:38:44,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:44,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:38:44,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:38:44,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:44,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:38:44,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:44,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:38:44,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:44,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:38:44,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:44,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:44,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:38:44,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:38:44,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:38:44,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:44,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:44,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:38:44,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:38:44,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:38:44,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:38:44,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:38:44,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt... +30: [2023-05-13 09:38:44,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt... +30: [2023-05-13 09:38:44,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt... +30: [2023-05-13 09:38:44,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:38:44,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:38:44,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:38:44,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:38:44,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:38:44,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:38:44,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:38:44,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:38:44,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt... +21: [2023-05-13 09:38:44,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt... +21: [2023-05-13 09:38:44,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt... +21: [2023-05-13 09:38:44,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt... +21: [2023-05-13 09:38:44,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:38:44,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:38:44,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:38:44,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:38:44,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt... +15: [2023-05-13 09:38:44,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt... +15: [2023-05-13 09:38:44,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt... +15: [2023-05-13 09:38:44,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt... +15: [2023-05-13 09:38:44,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt... +12: [2023-05-13 09:38:44,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt... +12: [2023-05-13 09:38:44,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt... +12: [2023-05-13 09:38:44,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt... +12: [2023-05-13 09:38:44,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt... +19: [2023-05-13 09:38:44,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt... +19: [2023-05-13 09:38:44,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt... +19: [2023-05-13 09:38:44,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt... +19: [2023-05-13 09:38:44,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:38:44,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:38:44,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:38:44,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:38:44,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt... + 2: [2023-05-13 09:38:44,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:38:44,431] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 21 + 2: [2023-05-13 09:38:44,438] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 21 +10: [2023-05-13 09:38:44,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:38:44,493] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 87 +10: [2023-05-13 09:38:44,499] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 87 + 2: [2023-05-13 09:38:44,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:38:44,562] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 19 + 2: [2023-05-13 09:38:44,572] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 19 + 4: [2023-05-13 09:38:44,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt. + 4: [2023-05-13 09:38:44,591] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 35 + 4: [2023-05-13 09:38:44,599] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 35 + 2: [2023-05-13 09:38:44,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:38:44,599] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 23 + 2: [2023-05-13 09:38:44,605] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 23 +25: [2023-05-13 09:38:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt. +25: [2023-05-13 09:38:44,619] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 205 +25: [2023-05-13 09:38:44,625] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 205 +31: [2023-05-13 09:38:44,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt. +31: [2023-05-13 09:38:44,628] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 251 +31: [2023-05-13 09:38:44,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt. +31: [2023-05-13 09:38:44,632] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 249 +31: [2023-05-13 09:38:44,635] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 251 +31: [2023-05-13 09:38:44,638] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 249 +28: [2023-05-13 09:38:44,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt. +28: [2023-05-13 09:38:44,643] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 229 +22: [2023-05-13 09:38:44,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt. +22: [2023-05-13 09:38:44,645] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 177 +13: [2023-05-13 09:38:44,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:38:44,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt. +13: [2023-05-13 09:38:44,647] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 105 + 2: [2023-05-13 09:38:44,647] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 17 +28: [2023-05-13 09:38:44,649] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 229 +22: [2023-05-13 09:38:44,652] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 177 +13: [2023-05-13 09:38:44,653] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 105 + 9: [2023-05-13 09:38:44,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt. + 9: [2023-05-13 09:38:44,654] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 75 +28: [2023-05-13 09:38:44,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:38:44,655] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 17 +28: [2023-05-13 09:38:44,655] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 227 +28: [2023-05-13 09:38:44,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt. +28: [2023-05-13 09:38:44,658] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 231 + 9: [2023-05-13 09:38:44,661] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 75 +28: [2023-05-13 09:38:44,661] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 227 +31: [2023-05-13 09:38:44,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt. +31: [2023-05-13 09:38:44,661] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 253 +28: [2023-05-13 09:38:44,663] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 231 +22: [2023-05-13 09:38:44,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt. +22: [2023-05-13 09:38:44,668] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 179 +31: [2023-05-13 09:38:44,669] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 253 +22: [2023-05-13 09:38:44,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt. +22: [2023-05-13 09:38:44,672] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 183 +22: [2023-05-13 09:38:44,674] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 179 + 9: [2023-05-13 09:38:44,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt. + 4: [2023-05-13 09:38:44,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt. + 9: [2023-05-13 09:38:44,677] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 73 + 4: [2023-05-13 09:38:44,677] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 37 +22: [2023-05-13 09:38:44,679] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 183 + 4: [2023-05-13 09:38:44,683] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 37 + 9: [2023-05-13 09:38:44,683] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 73 +26: [2023-05-13 09:38:44,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt. +26: [2023-05-13 09:38:44,685] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 213 +26: [2023-05-13 09:38:44,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt. +26: [2023-05-13 09:38:44,686] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 211 +24: [2023-05-13 09:38:44,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt. +24: [2023-05-13 09:38:44,690] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 195 +26: [2023-05-13 09:38:44,692] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 213 +26: [2023-05-13 09:38:44,693] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 211 +24: [2023-05-13 09:38:44,697] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 195 + 8: [2023-05-13 09:38:44,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt. + 8: [2023-05-13 09:38:44,706] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 65 + 8: [2023-05-13 09:38:44,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt. + 8: [2023-05-13 09:38:44,712] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 71 + 8: [2023-05-13 09:38:44,713] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 65 + 8: [2023-05-13 09:38:44,718] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 71 +14: [2023-05-13 09:38:44,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt. +14: [2023-05-13 09:38:44,721] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 115 +18: [2023-05-13 09:38:44,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:38:44,722] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 149 + 4: [2023-05-13 09:38:44,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt. + 4: [2023-05-13 09:38:44,724] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 33 + 8: [2023-05-13 09:38:44,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt. + 8: [2023-05-13 09:38:44,726] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 69 +14: [2023-05-13 09:38:44,728] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 115 + 5: [2023-05-13 09:38:44,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:38:44,728] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 41 +18: [2023-05-13 09:38:44,728] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 149 +21: [2023-05-13 09:38:44,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:38:44,730] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 175 + 4: [2023-05-13 09:38:44,731] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 33 + 5: [2023-05-13 09:38:44,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:38:44,732] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 45 + 8: [2023-05-13 09:38:44,732] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 69 +25: [2023-05-13 09:38:44,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt. +25: [2023-05-13 09:38:44,734] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 207 +16: [2023-05-13 09:38:44,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt. +22: [2023-05-13 09:38:44,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt. +16: [2023-05-13 09:38:44,734] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 129 +22: [2023-05-13 09:38:44,734] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 181 + 5: [2023-05-13 09:38:44,734] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 41 +21: [2023-05-13 09:38:44,736] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 175 + 5: [2023-05-13 09:38:44,739] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 45 +16: [2023-05-13 09:38:44,740] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 129 +25: [2023-05-13 09:38:44,740] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 207 +22: [2023-05-13 09:38:44,740] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 181 +24: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt. +24: [2023-05-13 09:38:44,743] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 193 + 4: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt. + 4: [2023-05-13 09:38:44,747] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 39 +24: [2023-05-13 09:38:44,749] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 193 + 7: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt. + 7: [2023-05-13 09:38:44,752] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 57 + 4: [2023-05-13 09:38:44,754] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 39 +20: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt. +20: [2023-05-13 09:38:44,757] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 163 + 7: [2023-05-13 09:38:44,758] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 57 +17: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt. +17: [2023-05-13 09:38:44,759] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 141 +13: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt. +13: [2023-05-13 09:38:44,762] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 109 +14: [2023-05-13 09:38:44,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt. +14: [2023-05-13 09:38:44,763] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 113 +20: [2023-05-13 09:38:44,763] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 163 +10: [2023-05-13 09:38:44,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:38:44,764] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 81 +17: [2023-05-13 09:38:44,766] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 141 + 9: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt. + 9: [2023-05-13 09:38:44,767] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 77 +13: [2023-05-13 09:38:44,768] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 109 +14: [2023-05-13 09:38:44,769] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 113 +10: [2023-05-13 09:38:44,771] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 81 + 9: [2023-05-13 09:38:44,774] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 77 +13: [2023-05-13 09:38:44,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt. +13: [2023-05-13 09:38:44,774] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 111 +28: [2023-05-13 09:38:44,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt. +28: [2023-05-13 09:38:44,780] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 225 +13: [2023-05-13 09:38:44,781] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 111 +28: [2023-05-13 09:38:44,786] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 225 + 8: [2023-05-13 09:38:44,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt. + 8: [2023-05-13 09:38:44,789] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 67 +20: [2023-05-13 09:38:44,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt. +20: [2023-05-13 09:38:44,791] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 161 + 3: [2023-05-13 09:38:44,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt. + 3: [2023-05-13 09:38:44,793] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 29 +16: [2023-05-13 09:38:44,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt. +16: [2023-05-13 09:38:44,794] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 131 + 8: [2023-05-13 09:38:44,796] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 67 +23: [2023-05-13 09:38:44,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:38:44,796] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 189 +20: [2023-05-13 09:38:44,798] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 161 + 3: [2023-05-13 09:38:44,799] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 29 +16: [2023-05-13 09:38:44,800] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 131 + 9: [2023-05-13 09:38:44,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:38:44,803] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 189 + 9: [2023-05-13 09:38:44,803] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 79 + 1: [2023-05-13 09:38:44,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:38:44,807] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 13 + 9: [2023-05-13 09:38:44,810] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 79 + 1: [2023-05-13 09:38:44,813] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 13 + 7: [2023-05-13 09:38:44,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt. + 7: [2023-05-13 09:38:44,815] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 59 +30: [2023-05-13 09:38:44,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt. +30: [2023-05-13 09:38:44,816] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 243 +25: [2023-05-13 09:38:44,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt. +25: [2023-05-13 09:38:44,818] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 201 + 7: [2023-05-13 09:38:44,820] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 59 +30: [2023-05-13 09:38:44,822] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 243 +21: [2023-05-13 09:38:44,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:38:44,824] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 171 +26: [2023-05-13 09:38:44,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt. +26: [2023-05-13 09:38:44,824] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 209 +25: [2023-05-13 09:38:44,824] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 201 +17: [2023-05-13 09:38:44,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt. +17: [2023-05-13 09:38:44,825] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 139 +17: [2023-05-13 09:38:44,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt. +17: [2023-05-13 09:38:44,829] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 143 +21: [2023-05-13 09:38:44,830] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 171 +26: [2023-05-13 09:38:44,830] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 209 +17: [2023-05-13 09:38:44,833] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 139 +15: [2023-05-13 09:38:44,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt. +15: [2023-05-13 09:38:44,834] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 127 + 0: [2023-05-13 09:38:44,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. + 0: [2023-05-13 09:38:44,835] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 7 +30: [2023-05-13 09:38:44,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt. +30: [2023-05-13 09:38:44,835] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 245 +17: [2023-05-13 09:38:44,835] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 143 + 5: [2023-05-13 09:38:44,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:38:44,837] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 47 + 5: [2023-05-13 09:38:44,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt. + 3: [2023-05-13 09:38:44,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:38:44,839] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 43 + 3: [2023-05-13 09:38:44,840] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 27 +15: [2023-05-13 09:38:44,840] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 127 +30: [2023-05-13 09:38:44,841] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 245 + 0: [2023-05-13 09:38:44,841] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 7 + 5: [2023-05-13 09:38:44,843] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 47 + 3: [2023-05-13 09:38:44,846] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 27 + 5: [2023-05-13 09:38:44,846] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 43 + 7: [2023-05-13 09:38:44,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt. + 7: [2023-05-13 09:38:44,847] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 61 +14: [2023-05-13 09:38:44,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt. +14: [2023-05-13 09:38:44,851] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 119 + 1: [2023-05-13 09:38:44,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:38:44,851] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 11 + 7: [2023-05-13 09:38:44,853] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 61 + 1: [2023-05-13 09:38:44,857] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 11 +14: [2023-05-13 09:38:44,858] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 119 +17: [2023-05-13 09:38:44,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt. +17: [2023-05-13 09:38:44,860] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 137 +12: [2023-05-13 09:38:44,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt. +12: [2023-05-13 09:38:44,861] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 101 +18: [2023-05-13 09:38:44,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:38:44,863] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 145 +21: [2023-05-13 09:38:44,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:38:44,865] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 169 +12: [2023-05-13 09:38:44,868] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 101 +17: [2023-05-13 09:38:44,868] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 137 +18: [2023-05-13 09:38:44,869] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 145 +21: [2023-05-13 09:38:44,872] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 169 + 3: [2023-05-13 09:38:44,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt. + 3: [2023-05-13 09:38:44,879] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 31 +25: [2023-05-13 09:38:44,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt. +25: [2023-05-13 09:38:44,882] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 203 + 3: [2023-05-13 09:38:44,885] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 31 + 1: [2023-05-13 09:38:44,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:38:44,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:38:44,886] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 9 +11: [2023-05-13 09:38:44,886] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 95 +23: [2023-05-13 09:38:44,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:38:44,887] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 187 +19: [2023-05-13 09:38:44,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt. +19: [2023-05-13 09:38:44,888] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 155 +15: [2023-05-13 09:38:44,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt. +25: [2023-05-13 09:38:44,889] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 203 +15: [2023-05-13 09:38:44,889] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 121 +18: [2023-05-13 09:38:44,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:38:44,892] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 147 +12: [2023-05-13 09:38:44,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:38:44,892] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 95 +12: [2023-05-13 09:38:44,892] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 99 +19: [2023-05-13 09:38:44,894] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 155 +23: [2023-05-13 09:38:44,894] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 187 + 1: [2023-05-13 09:38:44,895] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 9 +15: [2023-05-13 09:38:44,895] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 121 +12: [2023-05-13 09:38:44,898] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 99 +18: [2023-05-13 09:38:44,899] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 147 +20: [2023-05-13 09:38:44,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt. +20: [2023-05-13 09:38:44,900] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 165 + 1: [2023-05-13 09:38:44,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:38:44,904] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 15 +29: [2023-05-13 09:38:44,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt. +29: [2023-05-13 09:38:44,907] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 233 +20: [2023-05-13 09:38:44,908] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 165 + 1: [2023-05-13 09:38:44,911] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 15 +27: [2023-05-13 09:38:44,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt. +27: [2023-05-13 09:38:44,912] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 219 +29: [2023-05-13 09:38:44,914] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 233 +27: [2023-05-13 09:38:44,918] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 219 +12: [2023-05-13 09:38:44,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt. +12: [2023-05-13 09:38:44,924] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 97 +19: [2023-05-13 09:38:44,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt. +19: [2023-05-13 09:38:44,925] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 153 +21: [2023-05-13 09:38:44,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:38:44,925] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 173 +23: [2023-05-13 09:38:44,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:38:44,926] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 185 + 6: [2023-05-13 09:38:44,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt. + 6: [2023-05-13 09:38:44,927] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 49 +12: [2023-05-13 09:38:44,929] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 97 +29: [2023-05-13 09:38:44,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt. +29: [2023-05-13 09:38:44,930] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 235 +19: [2023-05-13 09:38:44,931] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 153 +15: [2023-05-13 09:38:44,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:38:44,932] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 185 +15: [2023-05-13 09:38:44,932] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 125 +21: [2023-05-13 09:38:44,932] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 173 + 6: [2023-05-13 09:38:44,934] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 49 + 6: [2023-05-13 09:38:44,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt. + 6: [2023-05-13 09:38:44,937] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 51 +29: [2023-05-13 09:38:44,937] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 235 +14: [2023-05-13 09:38:44,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt. +14: [2023-05-13 09:38:44,939] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 117 +15: [2023-05-13 09:38:44,940] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 125 +12: [2023-05-13 09:38:44,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt. +12: [2023-05-13 09:38:44,942] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 103 + 6: [2023-05-13 09:38:44,944] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 51 +14: [2023-05-13 09:38:44,945] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 117 +12: [2023-05-13 09:38:44,948] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 103 +11: [2023-05-13 09:38:44,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:38:44,950] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 91 +19: [2023-05-13 09:38:44,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt. +19: [2023-05-13 09:38:44,953] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 157 + 0: [2023-05-13 09:38:44,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. + 0: [2023-05-13 09:38:44,955] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 5 +20: [2023-05-13 09:38:44,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt. +20: [2023-05-13 09:38:44,956] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 167 +11: [2023-05-13 09:38:44,957] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 91 +19: [2023-05-13 09:38:44,958] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 157 + 0: [2023-05-13 09:38:44,961] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 5 +20: [2023-05-13 09:38:44,964] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 167 +31: [2023-05-13 09:38:44,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt. +31: [2023-05-13 09:38:44,969] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 255 + 0: [2023-05-13 09:38:44,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. + 0: [2023-05-13 09:38:44,974] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 3 +31: [2023-05-13 09:38:44,975] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 255 + 0: [2023-05-13 09:38:44,981] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 3 +24: [2023-05-13 09:38:44,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt. +24: [2023-05-13 09:38:44,986] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 199 +24: [2023-05-13 09:38:44,991] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 199 +15: [2023-05-13 09:38:45,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +15: [2023-05-13 09:38:45,001] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 126 +27: [2023-05-13 09:38:45,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt. +27: [2023-05-13 09:38:45,005] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 217 +15: [2023-05-13 09:38:45,008] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 126 +27: [2023-05-13 09:38:45,011] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 217 +24: [2023-05-13 09:38:45,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt. +24: [2023-05-13 09:38:45,024] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 197 +15: [2023-05-13 09:38:45,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt. +15: [2023-05-13 09:38:45,025] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 123 +27: [2023-05-13 09:38:45,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt. +27: [2023-05-13 09:38:45,026] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 223 +24: [2023-05-13 09:38:45,030] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 197 +15: [2023-05-13 09:38:45,032] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 123 +27: [2023-05-13 09:38:45,032] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 223 +13: [2023-05-13 09:38:45,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt. +13: [2023-05-13 09:38:45,059] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 107 +13: [2023-05-13 09:38:45,065] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 107 + 3: [2023-05-13 09:38:45,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt. + 3: [2023-05-13 09:38:45,067] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 25 +16: [2023-05-13 09:38:45,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt. +16: [2023-05-13 09:38:45,073] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 133 + 3: [2023-05-13 09:38:45,074] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 25 + 6: [2023-05-13 09:38:45,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt. + 6: [2023-05-13 09:38:45,076] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 53 +16: [2023-05-13 09:38:45,079] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 133 + 6: [2023-05-13 09:38:45,082] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 53 +23: [2023-05-13 09:38:45,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:38:45,098] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 191 +16: [2023-05-13 09:38:45,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt. +16: [2023-05-13 09:38:45,106] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 135 +23: [2023-05-13 09:38:45,106] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 191 +16: [2023-05-13 09:38:45,112] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 135 +26: [2023-05-13 09:38:45,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt. +26: [2023-05-13 09:38:45,131] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 215 +26: [2023-05-13 09:38:45,138] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 215 +18: [2023-05-13 09:38:45,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:38:45,179] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 151 +18: [2023-05-13 09:38:45,185] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 151 + 7: [2023-05-13 09:38:45,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt. + 7: [2023-05-13 09:38:45,186] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 63 + 7: [2023-05-13 09:38:45,195] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 63 +27: [2023-05-13 09:38:45,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt. +27: [2023-05-13 09:38:45,209] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 221 +27: [2023-05-13 09:38:45,216] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 221 + 0: [2023-05-13 09:38:45,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. + 0: [2023-05-13 09:38:45,324] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 1 + 0: [2023-05-13 09:38:45,330] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 1 +29: [2023-05-13 09:38:45,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt. +29: [2023-05-13 09:38:45,349] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 239 +29: [2023-05-13 09:38:45,356] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 239 +30: [2023-05-13 09:38:45,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt. +30: [2023-05-13 09:38:45,367] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 247 +30: [2023-05-13 09:38:45,373] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 247 +29: [2023-05-13 09:38:45,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt. +29: [2023-05-13 09:38:45,381] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 237 +29: [2023-05-13 09:38:45,389] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 237 +30: [2023-05-13 09:38:45,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt. +30: [2023-05-13 09:38:45,435] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 241 +30: [2023-05-13 09:38:45,440] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 241 + 4: [2023-05-13 09:38:46,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. + 4: [2023-05-13 09:38:46,266] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 32 + 4: [2023-05-13 09:38:46,277] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 32 +28: [2023-05-13 09:38:46,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:38:46,422] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 224 +28: [2023-05-13 09:38:46,428] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 224 +10: [2023-05-13 09:38:46,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:38:46,643] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 85 +10: [2023-05-13 09:38:46,649] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 85 +10: [2023-05-13 09:38:46,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:38:46,750] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 83 +10: [2023-05-13 09:38:46,756] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 83 + 6: [2023-05-13 09:38:47,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt. + 6: [2023-05-13 09:38:47,225] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 55 + 6: [2023-05-13 09:38:47,231] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 55 +19: [2023-05-13 09:38:47,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt. +19: [2023-05-13 09:38:47,250] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 159 +19: [2023-05-13 09:38:47,256] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 159 +11: [2023-05-13 09:38:47,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:38:47,469] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 93 +11: [2023-05-13 09:38:47,476] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 93 +11: [2023-05-13 09:38:47,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity25/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:38:47,516] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 89 +11: [2023-05-13 09:38:47,522] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 89 + 0: successfully loaded checkpoint from checkpoints_4b284bc4perplexity25 at iteration 0 +31: time (ms) | load-checkpoint: 19425.59 + 0: estimated model parameters: 4.24648704 + 0: estimated model parameters without embeddings: 4.078977024 + 0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-05-13 09:38:48 + 0: > building train, validation, and test datasets ... + 0: > datasets target sizes (minimum size): + 0: train: 1 + 0: validation: 102400 + 0: test: 102400 + 0: > building train, validation, and test datasets for GPT ... + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.053432 seconds + 0: number of documents: 3133972 + 0: > dataset split: + 0: train: + 0: document indices in [0, 3133972) total of 3133972 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.082 seconds + 0: total number of samples: 731002 + 0: total number of epochs: 1 + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.006074 seconds + 0: number of documents: 364608 + 0: > dataset split: + 0: validation: + 0: document indices in [0, 364608) total of 364608 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.076 seconds + 0: total number of samples: 169955 + 0: total number of epochs: 2 + 0: > finished creating GPT datasets ... + 0: [after dataloaders are built] datetime: 2023-05-13 09:38:53 + 0: done with setup ... + 0: training ... +31: time (ms) | model-and-optimizer-setup: 36161.43 | train/valid/test-data-iterators-setup: 4155.25 + 0: [after training is done] datetime: 2023-05-13 09:38:53 +31: ----------------------------------------------------------------------------------------------------------------- +31: validation loss at the end of training for val data | lm loss value: 2.624402E+00 | lm loss PPL: 1.379632E+01 | +31: ----------------------------------------------------------------------------------------------------------------- +END 3510898: Sat 13 May 2023 09:41:08 AM EEST diff --git a/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_0.json b/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..838eeb6d329b750c0c908b386e7583eedfa362fd --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.3056985698060885, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02037727591015296}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07003252145378697, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017782398203109468}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.29203668815673745, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00463136764747632}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.1041010135673265, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020426766002203623}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.0332378018219376, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0011349058010884613}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.13749957036557914, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003196748550458682}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.04860923158958796, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012225487600062404}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.06718224746092341, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0016048036895014346}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.284110989145015, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004532716214582295}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.10041814595908975, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0019026376308031996}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.06682529423751103, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016323321821141092}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.279078746288203, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004459389412998906}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.0995435574009657, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0019249335289248614}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_1.json b/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..34599bcf36fe32a8595f6f5da955a793b0f1f242 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.47118961796111253, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03698103082931082}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.0771933656618608, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015982731846139909}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.3718112290369406, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.005070882434958457}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.1177743328825882, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001968188676645897}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.036081371178737776, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001015398673414324}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.1783833017712426, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003533942569843169}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05451596763365407, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001220440063099909}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.07295630439059424, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014912307741854292}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.34998602256792577, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004635684392541653}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.11130332023816736, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018115727050860737}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07379279366484903, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015372531917501194}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.352791769527236, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004687142295790851}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.1123812114309373, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001868171085940279}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_2.json b/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..558cb7d7dc0b4f211b0fc4e663950224b383cba5 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.533916369829073, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03236121557558326}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07681910931236588, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015377577308842757}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.38510316670877215, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00510951140732995}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.11840000027377959, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018926983365336872}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.035783217444623636, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000976654693795426}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.18984984078562042, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0038247509890711063}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05478096245624173, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011716039831968148}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.0720673505406942, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014392854438264812}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.3590172306490562, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004627049792255802}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.1109268884100491, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017396910638209685}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07321894524691384, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014825495999582067}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.36495395152990295, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0047347356869845165}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.11265574136485931, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018002243925544347}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_3.json b/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..447813ad30bd6a9552d6173e311848fe2eb8fdaa --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.5595499974128665, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.037451927861185196}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.0779317898938085, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016112650938220434}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.38762436841783615, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0050758147406836035}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.1195630864031263, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.00192698764288485}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.03687332306481112, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010617608016665735}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.19234698280186005, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003750582513370695}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05609871570970779, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012335930575023653}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.073041109974632, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014857335049320345}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.36061760063023685, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004568863945261958}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.1120155525419754, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001772429101944478}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07415983300145405, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001517950201385796}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.3676109059529136, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004730150437103061}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.11375064173975076, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018326619772365102}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_4.json b/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..508bc2094401d2ca89f81f2efaabe63a748b546c --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.600990283111048, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.034277269466224586}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07788179321838232, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015013142830281594}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.39994843389146706, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00499808834668635}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.12037133162822398, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.00182661073083092}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.036729400826605405, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009707795653244158}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.20068990178179008, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0038369503757506584}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05640172523608126, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001156416498416956}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.0727796364763943, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014273675233445258}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.3686191869505379, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004446290243901058}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.11217146023041902, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001709041887848104}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07432987376050701, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014578128593730135}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.3785969497473814, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004640411180240199}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.11462098219560242, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017512279921868023}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_5.json b/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..e62bad06d2b1af9a0e4b332d90e896452db98447 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.6493610426477754, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04632676737275454}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07748205634617567, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001453985027637336}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.41054791184729794, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004995597119369392}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.12068693871021649, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017912623734371875}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.03639203356358758, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009275341362302266}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.20615921276358995, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0037483811580489817}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05650487878947583, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011343334162232442}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.0716220983354236, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013593103462355548}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.3738691644956053, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004316527656281062}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.11124019861288943, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016486705656890794}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07368867272301145, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014007140781525382}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.38732626297948775, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004596911969728994}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.11456244519233316, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017091885356239022}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_0.json b/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..39c14dc3acb69d47888ed0dd25db6f7776c561ce --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.141331703238122, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001900860421203431}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.24159721817911073, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002719597450403811}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.165509440090111, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001885757880608957}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.02669836310001385, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007156929489602658}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.04779652460533507, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013664090841134524}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.031517777959615835, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008078153472147894}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.10997003023895942, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013406611259386123}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.19583040922151732, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002241329057684982}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.1304904422471148, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013630786843314023}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.1296243531663802, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017327493840537764}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.2226802105122421, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002519165452551483}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.1520296742001466, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017239456047265416}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.3330159057137432, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0488742999692044}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_1.json b/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..a352bb9e0b3fafbc4a09662ec3b827445d542639 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.19155876571103367, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002235574181803164}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.31538809129393497, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027758196777476055}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.21776898864919678, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019444140343936993}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.04732904074441193, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010360766648087926}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.07972267565974463, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016475159287438514}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.05327544506657774, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009940356897765103}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.13475235285926426, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0015468525973368185}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.22894884785015995, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002173977332942091}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.1540568103772324, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0012940294587542422}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.17953406416101728, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0020944383459417203}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.29675466574734266, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002631907411205019}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.20425015913438996, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018113854464497062}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 2.6461954540180517, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0686607101579094}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_2.json b/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..f869f7e663f30a76b56a856c3ed999c0f57221d5 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.2159533628570682, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002791865511024776}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.30331115752873467, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002766040617891812}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.22064842886466463, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001912119893056796}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.05993156312422532, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0016401679053386375}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.08140349201064899, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016894533790392091}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.057864776743445145, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010769988684922042}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.1583382277260483, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0022683450533058772}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.22488747911377632, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002243037827514705}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.16022824117745368, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001353671145413634}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.20328152598542265, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002668656156379896}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.2859559600281066, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002647687376069602}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.20741540180362425, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001786464041080481}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 2.901173035530729, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05342955865413849}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_3.json b/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..730e7bbb36be3ab5980ecf289abf42cd6326681b --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.18489561929921422, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003137550370770306}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.24573209214855274, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003309411692082243}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.1795577823082448, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022472758798550777}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.05085763209734922, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0016343790426841317}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.06635653571954948, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016849742371905798}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.04723677657570569, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010843010239386048}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.13823892948584393, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002555329177823708}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.18353805339680163, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002620675134490683}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.13192124238531389, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001640698669618877}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.17421610987286962, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0029928898493940675}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.23156674787506518, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003147427389660552}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.16886339863190594, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021148662932854957}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 2.8536144517225503, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11123670367846313}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_4.json b/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..4aef240b34ec47ab20291da90f210290526459c7 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.05870358383616785, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0023430529684724016}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.08093434137408964, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0028367817835456407}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.05703592133005884, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001942831946209364}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.016328545160307874, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0011000353784742687}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.02180350520016562, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0011411604156949377}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.014808001674755853, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007238729065264632}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.044888286166910714, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001896455231106093}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.061444546156244624, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022065275178083242}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.04260019909764127, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014424624483373777}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.05510529257082465, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022153343918295677}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.07627563508448673, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026884645736360985}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.05352907711826245, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018257144932056115}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.5390525330469651, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0408109714920231}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_5.json b/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..39035772b54546bf543aaec4cb3cc02f6a482463 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.008584563764438044, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0008753066342352569}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.01304332354397915, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0012616701774741266}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.008834057407774154, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0008234294750612254}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.0020643566994266315, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0002999865807491949}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.003514834079841484, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00046085843512874496}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.0021982092172172316, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0002702560692849953}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.0065299312398118745, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0006600353263555559}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.010172425010043205, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0010133996136714378}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.006719281513421419, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0006225359173726148}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.007865790760235851, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0007878955854092591}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.012211564420719438, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0011927076918159406}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.008173501201069725, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0007588029707814247}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.1864794949178182e-06, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 2.2172651577024026e-06}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_0.json b/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e9d054caae70cda797f2cea70c0b5f5dcc6d81a7 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 4.765682760423997, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05422767541309084}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.17988772034955686, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016071110305014198}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.39581785916900913, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0029310989695743622}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.24332659914676713, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019769029717195945}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.07299811599029202, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009192181950106493}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.1655787870611749, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0019941826207487525}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.09956430250623664, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012064988114572355}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.1508460812164152, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012497621473603125}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.33638856499484304, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002462317442145038}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.2048927917623883, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015629945995320966}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.16020564267536394, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014848327378428146}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.35379035235820266, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0028050151273689558}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.21696200359533935, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018532122825693418}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_1.json b/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..28cb32457e501dbee0ca638f126c2f0ce40977e1 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 10.468883179786745, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.14513499791603968}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.4461560307039274, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0027882222137359425}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4223727993701862, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002779466413933148}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.41142927258080764, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020372645566246617}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.19138036565463667, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0021373415792973888}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.18105398962411778, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0020434906060036568}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.17546734923272342, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017675801853763418}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3234887060013004, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002422772594833981}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.3050111498702241, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0023058891158011475}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.29720682946492366, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018058222015338043}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.36955424114426116, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002627564057426632}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.35002355749667186, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025913666710418912}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3408169753521894, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0020222041567187483}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_2.json b/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..c2acfa5006c519f062946f1e16a048e968d7a07a --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 13.15937757187936, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.17402618441844517}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.5116677117570123, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003328551040183069}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4467016147483925, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0028150384160634126}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.44974386997659066, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021587456909989825}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.25116353200124875, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002675343325002614}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.21531631345677743, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0021418289381007287}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.21686040386653943, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0019364096185227812}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3875589106822081, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0029960409984381296}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.3362068824396353, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0024065124976756567}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3389361888586281, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001994011964068714}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.43414987277433165, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0032063239226803007}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.37817721348532046, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026544745921995615}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.38104681490559605, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021875307111738898}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_3.json b/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..b924676cf20b30a4c68ccb8c1b367fbc0dc45da4 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 12.129694716562593, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.09885740494020588}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.4445157890248481, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002595665567685936}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.47798126521143697, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027344918481641326}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4414312727012384, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019519907059499023}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.2128182501314406, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002080514621594728}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.23033986601501272, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002220133805877886}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.2107633646828048, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0018036725966383619}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.33200437713670644, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002293852098406482}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.35658301284722854, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0023733016537168627}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.32903441642513187, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001780398769078088}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.37782185980378685, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0025010018735451525}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.40662395947064806, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002656543475373511}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3753007198038583, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0020106738190462125}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_4.json b/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..42eb74839604c11aa5c56febd26576dd3863447b --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 12.132825433584408, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.16624627774955542}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.43551787708171735, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0022984268679554208}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.488774946659327, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0026403071353862067}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.44537146771645886, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001904249537543325}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.20679648264384795, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0018808563087133726}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.23542355545577054, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0022374175074990257}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.2120679639683205, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017947484063169713}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.32327382970798735, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001995377079294585}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.36425119832827263, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0023824504483212824}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.33096010620007504, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017733714402514843}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.37161856664153076, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022688211821483505}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.41744652285901757, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026213051295942776}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.380150071138794, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0019990851143011548}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_5.json b/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..c4892da2cfb5c7e9b3e130e3950b12de40febe23 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 12.150501863631494, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11314994370401227}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.43384350876692923, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0022430721979742672}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4930603001034525, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002670829206562973}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.44709276979411555, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001910905514583041}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.20616106117256064, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001830057336865641}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.23792096807821955, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00223350281305114}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.2132756026241378, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001789068514228881}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3221029481308322, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0019290523867159137}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.36806957352620984, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0024290518235640574}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3325234854628361, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017707439403097452}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.36995123008018044, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002227439242864291}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.42090674888926755, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002665490051406233}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.38137613302372186, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0020158755209571493}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_0.json b/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..d61f0098041edc487b24d5fafcea8b5a935d2fda --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.16949773638289, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0025008930161999152}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.35927434904922223, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004560400461326097}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.22169655207824404, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002737165165866136}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.0431701596864316, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001534312955547038}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.09474053242204084, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002989222654082048}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.056907827559785344, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0018251232736631654}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.1290524889805869, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020543657036148173}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.27361967658141045, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0035963924946506493}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.168504331059963, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0021699742342189397}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.13370844006346885, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0021670339989317035}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.285256434682157, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00408399090358691}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.1751085620216996, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0024047356803931223}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 2.3861113355807007, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11074519195435138}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_1.json b/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..0fd61619f1341c90bd7e82e5425e9eeeca414c0f --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.1463713881635103, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001933537484388361}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.35985532344291576, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0044711381442348866}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.20562256846157667, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0025917247878476176}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.036703622860321694, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001172772206888684}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.0939485415027206, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003047610569508944}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.052109640269165904, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0016499123215979108}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.1126929429063383, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014352356833621214}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.27967026507009923, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0035535952255309837}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.15866900668787276, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0019458892417091485}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.1163220909216439, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001603962253257027}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.2884084692973325, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003919667134306644}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.16374652691863595, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021813353055902}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 2.1599927233517997, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11630676231683935}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_2.json b/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..0451607d9ce9e87735abc9a89dc6a94cd0fa5ae6 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.14912207031164892, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001973097149015836}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.3654183781592464, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004588246988437046}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.20934999828708714, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0026521179445819754}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.038943395782983094, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0012528357050940226}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.09918868504865098, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003233414024371685}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0552385842587779, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017598952748467285}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.1156734461385958, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001526111534509325}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.28548150411787737, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003702367860147114}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.162664466797653, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002068423484247533}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.11782158647690419, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001648802010987179}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.29187107619025976, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004071548917864099}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.16589005254411232, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0022566042044053773}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 2.258614670093732, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08951840120004514}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_3.json b/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..1b2f1ef4f92eec2c7c51e711aa7f4de616295c90 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.14621341287651227, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0022083149046151504}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.3455642084346473, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004873293934425135}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.2009780831532814, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002806726765718572}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.037135604961774385, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0012274885414975227}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.09277441654545833, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0031426558683045865}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.05215230425231278, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017120956588686437}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.11361322830742238, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0016918706245087672}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.27017363415694445, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003905391020509053}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.15645205789193842, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0021779473947538264}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.11619780916522449, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001872558794638193}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.2770186544920654, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004312222731166786}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.16002294003908046, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002408602292236863}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 2.2258566055479094, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13278688909462125}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_4.json b/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..93ac9f9213b8e778c0974c16f47e60ff1e686ceb --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.04540495134964114, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002577404539511274}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.08944999300372532, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.005043063336771766}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.05602682791591825, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0030566032233541435}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.011337811755364056, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000979022200716812}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.02507829492852627, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002014742970966847}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.014711826789814367, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001150153366023636}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.03552240669275928, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020775346300726987}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.0694647286645038, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003953679220478554}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.04340501623577573, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002382694314714795}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.037416239535642694, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002189640562314706}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.07349320517885673, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004238026627054986}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.045785794288580685, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002531815611533499}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.1689695560959992, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.14941170439296517}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_5.json b/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..9b33a7715c91ffc042bc413cdec8e2ba16a0c479 --- /dev/null +++ b/perplexity25/evaluation/generation/agg.perplexity25_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.0033252126043761066, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0009227377103489034}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.0030093268798964366, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0008438210949622407}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.0031078634832113075, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0008672375952429783}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.0007101797265504257, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0003230925998744655}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.0006426199586576944, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0003070058829967824}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0006713467768799879, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0003135528125707898}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.0024626201177109525, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0007048985126880111}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.0021932876500475765, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0006418079510766518}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.002288473742428132, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0006642689228610262}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.0026697560249461646, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0007461825026075466}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.0023870797070471815, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0006822657353438714}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.0024871269676956433, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0007048637473097949}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 3.968509960352518e-38, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 9.071697715642431e-34}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_0.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0ab817625ba773440d920330e9db268d1f55e32c --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2da4b63d858d9213c3083caa6a1baf0e4faf7f432d84828d608e25dcafda642 +size 4181792 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_1.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..27c2d01701baa1811e9560cafd3b310effc32100 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08b1265893f493f03e8628c067fbba593e8f4640b6dfce911044effd8eab542d +size 5134919 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_2.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8c035b44844646856cc39d5e6422352c58d040bc --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a19a0d595df94347fa42f92001c485f029d162adf904c90a394ced134f225cdc +size 6029904 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_3.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a54e1142e0bed25a90bdc20ce289ea154cfea112 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d00b6ff6ffc96331d1a5cdf9c6a3ef234e21cd4ab9898aaec257181d9b71d0cc +size 6930952 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_4.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..47b37d16af66c1fbbc6c8313096686d37afea384 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81b35414f87ffd53e774b94b053fb1b3d841542dd14aa3464a4baef5fbafea60 +size 7845065 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_5.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f345387935fcff99080af70599a002c3eb686163 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_GEM-web_nlg_en_PALM_prompt_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d340f98212f01ba922ad6f36f8ae04119d20102bd2d2459228d4d02ab5b58b39 +size 8763343 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_0.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5660fdd376cdd38227128c0241bd13e2db1652b7 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e297694d235b5ea6112d76a2d7c3b91d2a2f962d8f1ab6553536c9a44e030cfc +size 7665925 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_1.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..50c1a3dc626c8f70795476500c140ee1ca85c0ea --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e79249be5d07ea37e59e7380372b06c8b2181806f4d4f383a587acafa5cc23c6 +size 13299982 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_2.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5d793769d7395621ceacaa499b84f584260f5edb --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42a68aeefc13e3caae308008b5d1e086d63e9cbc9fcf476c81c70eed172e4b94 +size 18857710 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_3.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b61e6221262ca05163c4dedc3139a32f33619ed9 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:387a74e556f3d59edcb574e91c01560cb7d71c7792cca9f2f97ddf3cbaa9b8e7 +size 24280471 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_4.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3bff60f85e31d5de5c7655e8dcd828c360be9011 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8498ef80ec45b85c7b28b92d798fca79346bbdf46f908bae955bf905676671cb +size 29459021 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_5.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..14dbdaeb0da22322d6fcc0b873ee7735a8f93d07 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_GEM-wiki_lingua_en_tldr_en_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67ec14b3607ec65bca8a4d98418327305c5ed55c3a2ad3fa680695965ade40a2 +size 34799000 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c03a11fa3bf426b2589813503397d39ed8d741ce --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec1cc2797dabf86af4224169f8f4f80fba26266f5787c03209afa27a011b7f91 +size 4515476 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..03c61ab77fefc177b0cbbb2411478548e8f53011 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1beb08293fb7ea81e39c4d1e6770395de52956bc6ef61594a648cb12e5bd1322 +size 5098060 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6430dda182a403fa926dfc42fc8b3f3b2a85c791 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b6f5894de4c8ca52279337196b36ee58a7281a594ab711569c15359321f6191 +size 6158026 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f49fc140c694a8e33c4ac1647ca4cbbe19ba63e3 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf8cfafe8a5e8c98b101321d09c00a1509fed0f8495640b0a1e5327ee629a508 +size 7319341 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1721700152d5d4c5198e69226f67727014146303 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f99067f0b3ffb9eb3736140b9dafac6ff4d8b34ab9e2ec7c89edcb62211a71c3 +size 8412109 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fc3a0b4c9819abf615db04609fc4ec6d6d7e4262 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d62204092f136f837351cb84d8185b721c26512200b894da3706335b81a39e46 +size 9504216 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_0.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..79a4418c8a829c67c5b1737f73072cec7c5781dc --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aa765a091917e79c8097f7bf71ae370b417c2927d61f9f026d89f5d173518b3 +size 2810040 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_1.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8e2f8e71484b50fa10b084e0e6ca871cd79b775b --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6fe9ecb1f525a4b92a96a2cbc6c3d47c4aef21ab902cc0cb9b9fccc3d12c147 +size 5106634 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_2.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b95940974c15b13015033c35e5087aaf51e66c60 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c61a1d1df848ef5e8625a021c7d3594e4c8c63c08ee509ccb65d1d1671746831 +size 7380654 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_3.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f85fb41d8e00a0e6e3f6d403baddad136d06d155 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1363a1431094fc9764053dc00336e1df5e499e4674b6435cbf5a8fd750c0c74 +size 9648403 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_4.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..839b22f5622d04579707a499093d1972eb4c5272 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46eb31746708bd4cd3ab46aff58aeab1c1b53eac147dc8b674c2b5053d3578bb +size 11672780 diff --git a/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_5.jsonl b/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a25c9029b9107b045ae5152d9ca75a6add897754 --- /dev/null +++ b/perplexity25/evaluation/generation/examples.perplexity25_gem_xsum_article_DOC_summary_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0fa60e6b18b1c01b1224aa1d3ed3d884ce24e863d7a2c46820bb4e2a9785974 +size 13897554 diff --git a/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_0.json b/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..39e8b5635ccbe27cca15bd042eb669beb581d9ab --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.3056985698060885, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.02037727591015296 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07003252145378697, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0017782398203109468 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.29203668815673745, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.00463136764747632 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.1041010135673265, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0020426766002203623 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.0332378018219376, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0011349058010884613 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.13749957036557914, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003196748550458682 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.04860923158958796, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0012225487600062404 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.06718224746092341, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0016048036895014346 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.284110989145015, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004532716214582295 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.10041814595908975, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0019026376308031996 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.06682529423751103, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0016323321821141092 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.279078746288203, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004459389412998906 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.0995435574009657, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0019249335289248614 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_1.json b/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..7cf4e57e9387438bf985810ceeffdd0001a60e81 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.47118961796111253, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.03698103082931082 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.0771933656618608, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0015982731846139909 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.3718112290369406, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.005070882434958457 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.1177743328825882, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001968188676645897 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.036081371178737776, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.001015398673414324 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.1783833017712426, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003533942569843169 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05451596763365407, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.001220440063099909 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.07295630439059424, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0014912307741854292 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.34998602256792577, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004635684392541653 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.11130332023816736, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0018115727050860737 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07379279366484903, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0015372531917501194 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.352791769527236, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004687142295790851 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.1123812114309373, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001868171085940279 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_2.json b/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..982f85c1f37be9649336deadeddc76180d8004a0 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.533916369829073, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.03236121557558326 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07681910931236588, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0015377577308842757 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.38510316670877215, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.00510951140732995 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.11840000027377959, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0018926983365336872 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.035783217444623636, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.000976654693795426 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.18984984078562042, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0038247509890711063 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05478096245624173, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011716039831968148 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.0720673505406942, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0014392854438264812 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.3590172306490562, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004627049792255802 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.1109268884100491, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0017396910638209685 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07321894524691384, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0014825495999582067 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.36495395152990295, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0047347356869845165 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.11265574136485931, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018002243925544347 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_3.json b/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..160e1ab320b239990f7b29bd5b2d5a3ba5973f99 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.5595499974128665, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.037451927861185196 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.0779317898938085, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0016112650938220434 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.38762436841783615, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0050758147406836035 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.1195630864031263, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.00192698764288485 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.03687332306481112, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0010617608016665735 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.19234698280186005, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003750582513370695 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05609871570970779, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0012335930575023653 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.073041109974632, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0014857335049320345 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.36061760063023685, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004568863945261958 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.1120155525419754, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001772429101944478 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07415983300145405, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001517950201385796 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.3676109059529136, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004730150437103061 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.11375064173975076, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018326619772365102 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_4.json b/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..e4afe1d06a4bb878cc031a90a6183ec92b42a0f5 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.600990283111048, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.034277269466224586 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07788179321838232, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0015013142830281594 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.39994843389146706, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.00499808834668635 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.12037133162822398, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.00182661073083092 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.036729400826605405, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0009707795653244158 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.20068990178179008, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0038369503757506584 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05640172523608126, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.001156416498416956 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.0727796364763943, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0014273675233445258 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.3686191869505379, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004446290243901058 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.11217146023041902, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001709041887848104 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07432987376050701, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0014578128593730135 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.3785969497473814, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004640411180240199 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.11462098219560242, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0017512279921868023 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_5.json b/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..b5c18a5ba78555eda5d597816c38cff2dbaaa3c9 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.6493610426477754, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.04632676737275454 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07748205634617567, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001453985027637336 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.41054791184729794, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004995597119369392 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.12068693871021649, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0017912623734371875 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.03639203356358758, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0009275341362302266 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.20615921276358995, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0037483811580489817 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05650487878947583, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011343334162232442 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.0716220983354236, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0013593103462355548 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.3738691644956053, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004316527656281062 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.11124019861288943, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0016486705656890794 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07368867272301145, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0014007140781525382 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.38732626297948775, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004596911969728994 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.11456244519233316, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0017091885356239022 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_0.json b/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..7fd66809a14172d98708741cd96413facd744aec --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.141331703238122, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001900860421203431 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.24159721817911073, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.002719597450403811 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.165509440090111, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001885757880608957 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.02669836310001385, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007156929489602658 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.04779652460533507, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0013664090841134524 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.031517777959615835, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0008078153472147894 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.10997003023895942, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0013406611259386123 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.19583040922151732, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.002241329057684982 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.1304904422471148, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0013630786843314023 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.1296243531663802, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0017327493840537764 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.2226802105122421, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.002519165452551483 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.1520296742001466, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0017239456047265416 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.3330159057137432, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.0488742999692044 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_1.json b/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..f631aa047748428bfd88237e0089b9650e294265 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.19155876571103367, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.002235574181803164 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.31538809129393497, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0027758196777476055 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.21776898864919678, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0019444140343936993 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.04732904074441193, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0010360766648087926 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.07972267565974463, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0016475159287438514 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.05327544506657774, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0009940356897765103 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.13475235285926426, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0015468525973368185 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.22894884785015995, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.002173977332942091 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.1540568103772324, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0012940294587542422 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.17953406416101728, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0020944383459417203 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.29675466574734266, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.002631907411205019 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.20425015913438996, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018113854464497062 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 2.6461954540180517, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.0686607101579094 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_2.json b/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..1a7a10b5684dd6ee8fdd14cb2b8eb4214bf6bcbe --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.2159533628570682, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.002791865511024776 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.30331115752873467, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.002766040617891812 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.22064842886466463, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001912119893056796 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.05993156312422532, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0016401679053386375 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.08140349201064899, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0016894533790392091 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.057864776743445145, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010769988684922042 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.1583382277260483, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0022683450533058772 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.22488747911377632, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.002243037827514705 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.16022824117745368, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001353671145413634 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.20328152598542265, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.002668656156379896 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.2859559600281066, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.002647687376069602 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.20741540180362425, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001786464041080481 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 2.901173035530729, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.05342955865413849 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_3.json b/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..5efba2156c589c3f5f8981714cd66d3b934edef2 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.18489561929921422, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.003137550370770306 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.24573209214855274, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.003309411692082243 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.1795577823082448, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0022472758798550777 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.05085763209734922, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0016343790426841317 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.06635653571954948, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0016849742371905798 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.04723677657570569, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010843010239386048 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.13823892948584393, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.002555329177823708 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.18353805339680163, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.002620675134490683 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.13192124238531389, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001640698669618877 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.17421610987286962, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0029928898493940675 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.23156674787506518, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.003147427389660552 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.16886339863190594, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0021148662932854957 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 2.8536144517225503, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.11123670367846313 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_4.json b/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..3cb3c69bdf1599b28ffa2c2410672a17128dd724 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.05870358383616785, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0023430529684724016 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.08093434137408964, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0028367817835456407 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.05703592133005884, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001942831946209364 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.016328545160307874, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0011000353784742687 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.02180350520016562, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0011411604156949377 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.014808001674755853, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0007238729065264632 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.044888286166910714, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.001896455231106093 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.061444546156244624, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0022065275178083242 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.04260019909764127, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0014424624483373777 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.05510529257082465, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0022153343918295677 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.07627563508448673, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0026884645736360985 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.05352907711826245, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018257144932056115 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.5390525330469651, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.0408109714920231 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_5.json b/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..9a4cab9404d44c9a54330a344c30a8b7df5198ce --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.008584563764438044, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0008753066342352569 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.01304332354397915, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0012616701774741266 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.008834057407774154, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0008234294750612254 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.0020643566994266315, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0002999865807491949 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.003514834079841484, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.00046085843512874496 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.0021982092172172316, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0002702560692849953 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.0065299312398118745, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0006600353263555559 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.010172425010043205, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0010133996136714378 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.006719281513421419, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0006225359173726148 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.007865790760235851, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0007878955854092591 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.012211564420719438, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0011927076918159406 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.008173501201069725, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0007588029707814247 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.1864794949178182e-06, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 2.2172651577024026e-06 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_0.json b/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..a16a00632b8f4d8497664343d0acdceed5ef3ed5 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 4.765682760423997, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.05422767541309084 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.17988772034955686, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0016071110305014198 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.39581785916900913, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0029310989695743622 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.24332659914676713, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0019769029717195945 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.07299811599029202, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0009192181950106493 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.1655787870611749, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0019941826207487525 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.09956430250623664, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0012064988114572355 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.1508460812164152, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0012497621473603125 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.33638856499484304, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002462317442145038 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.2048927917623883, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0015629945995320966 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.16020564267536394, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0014848327378428146 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.35379035235820266, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0028050151273689558 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.21696200359533935, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018532122825693418 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_1.json b/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..200fd205ec54c619e5affee3748844d37f71d910 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 10.468883179786745, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.14513499791603968 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.4461560307039274, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0027882222137359425 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4223727993701862, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002779466413933148 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.41142927258080764, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0020372645566246617 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.19138036565463667, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0021373415792973888 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.18105398962411778, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0020434906060036568 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.17546734923272342, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0017675801853763418 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3234887060013004, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002422772594833981 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.3050111498702241, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0023058891158011475 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.29720682946492366, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0018058222015338043 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.36955424114426116, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.002627564057426632 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.35002355749667186, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0025913666710418912 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3408169753521894, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0020222041567187483 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_2.json b/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..ef045050e65cd07b4fbf87902f42a991d81cab72 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 13.15937757187936, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.17402618441844517 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.5116677117570123, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003328551040183069 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4467016147483925, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0028150384160634126 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.44974386997659066, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0021587456909989825 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.25116353200124875, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002675343325002614 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.21531631345677743, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0021418289381007287 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.21686040386653943, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0019364096185227812 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3875589106822081, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0029960409984381296 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.3362068824396353, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0024065124976756567 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3389361888586281, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.001994011964068714 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.43414987277433165, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0032063239226803007 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.37817721348532046, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0026544745921995615 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.38104681490559605, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0021875307111738898 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_3.json b/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..90dd70e570f9f440e86016d7db0a94a47fe44995 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 12.129694716562593, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.09885740494020588 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.4445157890248481, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.002595665567685936 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.47798126521143697, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0027344918481641326 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4414312727012384, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0019519907059499023 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.2128182501314406, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002080514621594728 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.23033986601501272, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.002220133805877886 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.2107633646828048, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0018036725966383619 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.33200437713670644, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002293852098406482 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.35658301284722854, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0023733016537168627 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.32903441642513187, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.001780398769078088 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.37782185980378685, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0025010018735451525 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.40662395947064806, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002656543475373511 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3753007198038583, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0020106738190462125 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_4.json b/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c91f511cca7cc98626b40cafb5f2bb87a87fbf16 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 12.132825433584408, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.16624627774955542 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.43551787708171735, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0022984268679554208 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.488774946659327, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0026403071353862067 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.44537146771645886, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.001904249537543325 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.20679648264384795, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0018808563087133726 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.23542355545577054, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0022374175074990257 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.2120679639683205, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0017947484063169713 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.32327382970798735, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.001995377079294585 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.36425119832827263, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0023824504483212824 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.33096010620007504, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0017733714402514843 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.37161856664153076, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0022688211821483505 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.41744652285901757, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0026213051295942776 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.380150071138794, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0019990851143011548 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_5.json b/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..8174c8404f23ccb86b72358891857146743182da --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 12.150501863631494, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.11314994370401227 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.43384350876692923, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0022430721979742672 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4930603001034525, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002670829206562973 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.44709276979411555, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.001910905514583041 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.20616106117256064, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.001830057336865641 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.23792096807821955, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.00223350281305114 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.2132756026241378, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.001789068514228881 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3221029481308322, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0019290523867159137 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.36806957352620984, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0024290518235640574 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3325234854628361, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0017707439403097452 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.36995123008018044, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.002227439242864291 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.42090674888926755, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002665490051406233 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.38137613302372186, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0020158755209571493 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_0.json b/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..43b1437d5bf165044fa93f98854de6a25c502f93 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.16949773638289, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0025008930161999152 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.35927434904922223, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004560400461326097 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.22169655207824404, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002737165165866136 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.0431701596864316, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.001534312955547038 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.09474053242204084, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.002989222654082048 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.056907827559785344, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0018251232736631654 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.1290524889805869, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0020543657036148173 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.27361967658141045, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0035963924946506493 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.168504331059963, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0021699742342189397 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.13370844006346885, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0021670339989317035 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.285256434682157, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.00408399090358691 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.1751085620216996, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0024047356803931223 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 2.3861113355807007, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.11074519195435138 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_1.json b/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..de07aa93ebf35c724f1e8ad6e531e859b7a520d9 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.1463713881635103, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.001933537484388361 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.35985532344291576, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0044711381442348866 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.20562256846157667, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0025917247878476176 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.036703622860321694, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.001172772206888684 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.0939485415027206, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.003047610569508944 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.052109640269165904, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0016499123215979108 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.1126929429063383, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0014352356833621214 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.27967026507009923, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0035535952255309837 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.15866900668787276, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0019458892417091485 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.1163220909216439, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.001603962253257027 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.2884084692973325, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.003919667134306644 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.16374652691863595, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0021813353055902 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 2.1599927233517997, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.11630676231683935 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_2.json b/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..d70e2672bcdcb6730cfa1d2706efa45d2cfe5fd9 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.14912207031164892, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.001973097149015836 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.3654183781592464, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004588246988437046 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.20934999828708714, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0026521179445819754 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.038943395782983094, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0012528357050940226 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.09918868504865098, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.003233414024371685 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.0552385842587779, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0017598952748467285 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.1156734461385958, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.001526111534509325 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.28548150411787737, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.003702367860147114 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.162664466797653, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.002068423484247533 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.11782158647690419, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.001648802010987179 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.29187107619025976, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.004071548917864099 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.16589005254411232, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0022566042044053773 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 2.258614670093732, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.08951840120004514 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_3.json b/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..568ed5994122647819e1eabaf8c34d33af8971a0 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.14621341287651227, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0022083149046151504 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.3455642084346473, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004873293934425135 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.2009780831532814, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002806726765718572 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.037135604961774385, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0012274885414975227 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.09277441654545833, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0031426558683045865 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.05215230425231278, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0017120956588686437 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.11361322830742238, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0016918706245087672 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.27017363415694445, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.003905391020509053 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.15645205789193842, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0021779473947538264 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.11619780916522449, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.001872558794638193 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.2770186544920654, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.004312222731166786 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.16002294003908046, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.002408602292236863 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 2.2258566055479094, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.13278688909462125 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_4.json b/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..cb91c8998c9bf8d0c8283774e017a72cd197a520 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.04540495134964114, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.002577404539511274 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.08944999300372532, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.005043063336771766 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.05602682791591825, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0030566032233541435 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.011337811755364056, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.000979022200716812 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.02507829492852627, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.002014742970966847 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.014711826789814367, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.001150153366023636 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.03552240669275928, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0020775346300726987 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.0694647286645038, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.003953679220478554 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.04340501623577573, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.002382694314714795 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.037416239535642694, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.002189640562314706 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.07349320517885673, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.004238026627054986 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.045785794288580685, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.002531815611533499 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.1689695560959992, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.14941170439296517 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_5.json b/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..a5a550f5aaac83ceb164a8cc8904acdc229ccc07 --- /dev/null +++ b/perplexity25/evaluation/generation/slim.perplexity25_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.0033252126043761066, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0009227377103489034 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.0030093268798964366, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0008438210949622407 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.0031078634832113075, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0008672375952429783 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.0007101797265504257, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0003230925998744655 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.0006426199586576944, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0003070058829967824 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.0006713467768799879, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0003135528125707898 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.0024626201177109525, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0007048985126880111 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.0021932876500475765, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0006418079510766518 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.002288473742428132, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0006642689228610262 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.0026697560249461646, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0007461825026075466 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.0023870797070471815, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0006822657353438714 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.0024871269676956433, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0007048637473097949 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 3.968509960352518e-38, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 9.071697715642431e-34 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/rankeval/perplexity25_0.json b/perplexity25/evaluation/rankeval/perplexity25_0.json new file mode 100644 index 0000000000000000000000000000000000000000..5e0f51e5b61758e4c5f39bc32aece0071e60db8d --- /dev/null +++ b/perplexity25/evaluation/rankeval/perplexity25_0.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.338, + "acc_stderr": 0.014965960710224482 + }, + "anli_r2": { + "acc": 0.341, + "acc_stderr": 0.014998131348402699 + }, + "anli_r3": { + "acc": 0.3491666666666667, + "acc_stderr": 0.013767075395077249 + }, + "cb": { + "acc": 0.5, + "acc_stderr": 0.06741998624632421, + "f1": 0.33763440860215055 + }, + "copa": { + "acc": 0.76, + "acc_stderr": 0.04292346959909283 + }, + "hellaswag": { + "acc": 0.4814777932682733, + "acc_stderr": 0.004986356526063966, + "acc_norm": 0.6296554471220872, + "acc_norm_stderr": 0.004819100456867812 + }, + "rte": { + "acc": 0.5884476534296029, + "acc_stderr": 0.029621832222417196 + }, + "winogrande": { + "acc": 0.611681136543015, + "acc_stderr": 0.01369745665845723 + }, + "storycloze_2016": { + "acc": 0.740780331373597, + "acc_stderr": 0.010133463176449564 + }, + "boolq": { + "acc": 0.5972477064220183, + "acc_stderr": 0.008578054401368405 + }, + "arc_easy": { + "acc": 0.6430976430976431, + "acc_stderr": 0.009830630210347005, + "acc_norm": 0.5686026936026936, + "acc_norm_stderr": 0.010162752847747505 + }, + "arc_challenge": { + "acc": 0.3054607508532423, + "acc_stderr": 0.013460080478002501, + "acc_norm": 0.3267918088737201, + "acc_norm_stderr": 0.013706665975587336 + }, + "sciq": { + "acc": 0.869, + "acc_stderr": 0.010674874844837952, + "acc_norm": 0.773, + "acc_norm_stderr": 0.013253174964763921 + }, + "piqa": { + "acc": 0.7415669205658324, + "acc_stderr": 0.010213971636773319, + "acc_norm": 0.7421109902067464, + "acc_norm_stderr": 0.010206956662056252 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/rankeval/perplexity25_0_lm-eval_global_step80108_2023-05-13-09-53-07_0shots_backup.json b/perplexity25/evaluation/rankeval/perplexity25_0_lm-eval_global_step80108_2023-05-13-09-53-07_0shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..5e0f51e5b61758e4c5f39bc32aece0071e60db8d --- /dev/null +++ b/perplexity25/evaluation/rankeval/perplexity25_0_lm-eval_global_step80108_2023-05-13-09-53-07_0shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.338, + "acc_stderr": 0.014965960710224482 + }, + "anli_r2": { + "acc": 0.341, + "acc_stderr": 0.014998131348402699 + }, + "anli_r3": { + "acc": 0.3491666666666667, + "acc_stderr": 0.013767075395077249 + }, + "cb": { + "acc": 0.5, + "acc_stderr": 0.06741998624632421, + "f1": 0.33763440860215055 + }, + "copa": { + "acc": 0.76, + "acc_stderr": 0.04292346959909283 + }, + "hellaswag": { + "acc": 0.4814777932682733, + "acc_stderr": 0.004986356526063966, + "acc_norm": 0.6296554471220872, + "acc_norm_stderr": 0.004819100456867812 + }, + "rte": { + "acc": 0.5884476534296029, + "acc_stderr": 0.029621832222417196 + }, + "winogrande": { + "acc": 0.611681136543015, + "acc_stderr": 0.01369745665845723 + }, + "storycloze_2016": { + "acc": 0.740780331373597, + "acc_stderr": 0.010133463176449564 + }, + "boolq": { + "acc": 0.5972477064220183, + "acc_stderr": 0.008578054401368405 + }, + "arc_easy": { + "acc": 0.6430976430976431, + "acc_stderr": 0.009830630210347005, + "acc_norm": 0.5686026936026936, + "acc_norm_stderr": 0.010162752847747505 + }, + "arc_challenge": { + "acc": 0.3054607508532423, + "acc_stderr": 0.013460080478002501, + "acc_norm": 0.3267918088737201, + "acc_norm_stderr": 0.013706665975587336 + }, + "sciq": { + "acc": 0.869, + "acc_stderr": 0.010674874844837952, + "acc_norm": 0.773, + "acc_norm_stderr": 0.013253174964763921 + }, + "piqa": { + "acc": 0.7415669205658324, + "acc_stderr": 0.010213971636773319, + "acc_norm": 0.7421109902067464, + "acc_norm_stderr": 0.010206956662056252 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/rankeval/perplexity25_1.json b/perplexity25/evaluation/rankeval/perplexity25_1.json new file mode 100644 index 0000000000000000000000000000000000000000..0ded4a4904443b8dac152af96a63e564a75a270f --- /dev/null +++ b/perplexity25/evaluation/rankeval/perplexity25_1.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.329, + "acc_stderr": 0.014865395385928364 + }, + "anli_r2": { + "acc": 0.323, + "acc_stderr": 0.01479492784334864 + }, + "anli_r3": { + "acc": 0.3425, + "acc_stderr": 0.013704669762934725 + }, + "cb": { + "acc": 0.5178571428571429, + "acc_stderr": 0.06737697508644648, + "f1": 0.36209571547917413 + }, + "copa": { + "acc": 0.8, + "acc_stderr": 0.040201512610368445 + }, + "hellaswag": { + "acc": 0.4794861581358295, + "acc_stderr": 0.0049855800659464565, + "acc_norm": 0.6293567018522207, + "acc_norm_stderr": 0.00481989994534249 + }, + "rte": { + "acc": 0.5703971119133574, + "acc_stderr": 0.02979666882912467 + }, + "winogrande": { + "acc": 0.6148382004735596, + "acc_stderr": 0.013676821287521412 + }, + "storycloze_2016": { + "acc": 0.7295563869588455, + "acc_stderr": 0.010271810373331027 + }, + "boolq": { + "acc": 0.6412844036697247, + "acc_stderr": 0.0083886680340594 + }, + "arc_easy": { + "acc": 0.6632996632996633, + "acc_stderr": 0.009697166595752472, + "acc_norm": 0.6397306397306397, + "acc_norm_stderr": 0.00985100258473238 + }, + "arc_challenge": { + "acc": 0.3310580204778157, + "acc_stderr": 0.01375206241981784, + "acc_norm": 0.3575085324232082, + "acc_norm_stderr": 0.014005494275916573 + }, + "sciq": { + "acc": 0.916, + "acc_stderr": 0.008776162089491115, + "acc_norm": 0.899, + "acc_norm_stderr": 0.009533618929340987 + }, + "piqa": { + "acc": 0.7388465723612623, + "acc_stderr": 0.010248738649935578, + "acc_norm": 0.7301414581066377, + "acc_norm_stderr": 0.010356595421852195 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/rankeval/perplexity25_1_lm-eval_global_step80108_2023-05-13-09-53-07_1shots_backup.json b/perplexity25/evaluation/rankeval/perplexity25_1_lm-eval_global_step80108_2023-05-13-09-53-07_1shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..0ded4a4904443b8dac152af96a63e564a75a270f --- /dev/null +++ b/perplexity25/evaluation/rankeval/perplexity25_1_lm-eval_global_step80108_2023-05-13-09-53-07_1shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.329, + "acc_stderr": 0.014865395385928364 + }, + "anli_r2": { + "acc": 0.323, + "acc_stderr": 0.01479492784334864 + }, + "anli_r3": { + "acc": 0.3425, + "acc_stderr": 0.013704669762934725 + }, + "cb": { + "acc": 0.5178571428571429, + "acc_stderr": 0.06737697508644648, + "f1": 0.36209571547917413 + }, + "copa": { + "acc": 0.8, + "acc_stderr": 0.040201512610368445 + }, + "hellaswag": { + "acc": 0.4794861581358295, + "acc_stderr": 0.0049855800659464565, + "acc_norm": 0.6293567018522207, + "acc_norm_stderr": 0.00481989994534249 + }, + "rte": { + "acc": 0.5703971119133574, + "acc_stderr": 0.02979666882912467 + }, + "winogrande": { + "acc": 0.6148382004735596, + "acc_stderr": 0.013676821287521412 + }, + "storycloze_2016": { + "acc": 0.7295563869588455, + "acc_stderr": 0.010271810373331027 + }, + "boolq": { + "acc": 0.6412844036697247, + "acc_stderr": 0.0083886680340594 + }, + "arc_easy": { + "acc": 0.6632996632996633, + "acc_stderr": 0.009697166595752472, + "acc_norm": 0.6397306397306397, + "acc_norm_stderr": 0.00985100258473238 + }, + "arc_challenge": { + "acc": 0.3310580204778157, + "acc_stderr": 0.01375206241981784, + "acc_norm": 0.3575085324232082, + "acc_norm_stderr": 0.014005494275916573 + }, + "sciq": { + "acc": 0.916, + "acc_stderr": 0.008776162089491115, + "acc_norm": 0.899, + "acc_norm_stderr": 0.009533618929340987 + }, + "piqa": { + "acc": 0.7388465723612623, + "acc_stderr": 0.010248738649935578, + "acc_norm": 0.7301414581066377, + "acc_norm_stderr": 0.010356595421852195 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/rankeval/perplexity25_2.json b/perplexity25/evaluation/rankeval/perplexity25_2.json new file mode 100644 index 0000000000000000000000000000000000000000..68756d170fdc5908084deddd24a7e033e03720f0 --- /dev/null +++ b/perplexity25/evaluation/rankeval/perplexity25_2.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.33, + "acc_stderr": 0.014876872027456732 + }, + "anli_r2": { + "acc": 0.33, + "acc_stderr": 0.014876872027456736 + }, + "anli_r3": { + "acc": 0.33166666666666667, + "acc_stderr": 0.01359683672948516 + }, + "cb": { + "acc": 0.5178571428571429, + "acc_stderr": 0.06737697508644648, + "f1": 0.35968427443837275 + }, + "copa": { + "acc": 0.79, + "acc_stderr": 0.040936018074033256 + }, + "hellaswag": { + "acc": 0.4790878311093408, + "acc_stderr": 0.004985415250690917, + "acc_norm": 0.6304521011750648, + "acc_norm_stderr": 0.004816958817726088 + }, + "rte": { + "acc": 0.5703971119133574, + "acc_stderr": 0.02979666882912467 + }, + "winogrande": { + "acc": 0.6235201262825573, + "acc_stderr": 0.013616931960667187 + }, + "storycloze_2016": { + "acc": 0.7365045430251203, + "acc_stderr": 0.010187168219156485 + }, + "boolq": { + "acc": 0.6428134556574924, + "acc_stderr": 0.008380743796951404 + }, + "arc_easy": { + "acc": 0.6742424242424242, + "acc_stderr": 0.009616642976885964, + "acc_norm": 0.6523569023569024, + "acc_norm_stderr": 0.009771868846830909 + }, + "arc_challenge": { + "acc": 0.3378839590443686, + "acc_stderr": 0.013822047922283507, + "acc_norm": 0.3660409556313993, + "acc_norm_stderr": 0.014077223108470144 + }, + "sciq": { + "acc": 0.923, + "acc_stderr": 0.008434580140240644, + "acc_norm": 0.901, + "acc_norm_stderr": 0.009449248027662746 + }, + "piqa": { + "acc": 0.7404787812840044, + "acc_stderr": 0.010227939888173918, + "acc_norm": 0.7388465723612623, + "acc_norm_stderr": 0.010248738649935587 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/rankeval/perplexity25_2_lm-eval_global_step80108_2023-05-13-09-53-07_2shots_backup.json b/perplexity25/evaluation/rankeval/perplexity25_2_lm-eval_global_step80108_2023-05-13-09-53-07_2shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..68756d170fdc5908084deddd24a7e033e03720f0 --- /dev/null +++ b/perplexity25/evaluation/rankeval/perplexity25_2_lm-eval_global_step80108_2023-05-13-09-53-07_2shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.33, + "acc_stderr": 0.014876872027456732 + }, + "anli_r2": { + "acc": 0.33, + "acc_stderr": 0.014876872027456736 + }, + "anli_r3": { + "acc": 0.33166666666666667, + "acc_stderr": 0.01359683672948516 + }, + "cb": { + "acc": 0.5178571428571429, + "acc_stderr": 0.06737697508644648, + "f1": 0.35968427443837275 + }, + "copa": { + "acc": 0.79, + "acc_stderr": 0.040936018074033256 + }, + "hellaswag": { + "acc": 0.4790878311093408, + "acc_stderr": 0.004985415250690917, + "acc_norm": 0.6304521011750648, + "acc_norm_stderr": 0.004816958817726088 + }, + "rte": { + "acc": 0.5703971119133574, + "acc_stderr": 0.02979666882912467 + }, + "winogrande": { + "acc": 0.6235201262825573, + "acc_stderr": 0.013616931960667187 + }, + "storycloze_2016": { + "acc": 0.7365045430251203, + "acc_stderr": 0.010187168219156485 + }, + "boolq": { + "acc": 0.6428134556574924, + "acc_stderr": 0.008380743796951404 + }, + "arc_easy": { + "acc": 0.6742424242424242, + "acc_stderr": 0.009616642976885964, + "acc_norm": 0.6523569023569024, + "acc_norm_stderr": 0.009771868846830909 + }, + "arc_challenge": { + "acc": 0.3378839590443686, + "acc_stderr": 0.013822047922283507, + "acc_norm": 0.3660409556313993, + "acc_norm_stderr": 0.014077223108470144 + }, + "sciq": { + "acc": 0.923, + "acc_stderr": 0.008434580140240644, + "acc_norm": 0.901, + "acc_norm_stderr": 0.009449248027662746 + }, + "piqa": { + "acc": 0.7404787812840044, + "acc_stderr": 0.010227939888173918, + "acc_norm": 0.7388465723612623, + "acc_norm_stderr": 0.010248738649935587 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/rankeval/perplexity25_3.json b/perplexity25/evaluation/rankeval/perplexity25_3.json new file mode 100644 index 0000000000000000000000000000000000000000..b359438b1b5373f8773ccb39c348ad21032b994f --- /dev/null +++ b/perplexity25/evaluation/rankeval/perplexity25_3.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.324, + "acc_stderr": 0.014806864733738859 + }, + "anli_r2": { + "acc": 0.322, + "acc_stderr": 0.014782913600996676 + }, + "anli_r3": { + "acc": 0.33166666666666667, + "acc_stderr": 0.013596836729485163 + }, + "cb": { + "acc": 0.5, + "acc_stderr": 0.06741998624632421, + "f1": 0.32868937048503616 + }, + "copa": { + "acc": 0.81, + "acc_stderr": 0.03942772444036623 + }, + "hellaswag": { + "acc": 0.4817765385381398, + "acc_stderr": 0.00498646615169878, + "acc_norm": 0.6341366261700856, + "acc_norm_stderr": 0.004806870285747301 + }, + "rte": { + "acc": 0.5451263537906137, + "acc_stderr": 0.029973636495415255 + }, + "winogrande": { + "acc": 0.6290449881610103, + "acc_stderr": 0.01357639990223157 + }, + "storycloze_2016": { + "acc": 0.7439871726349546, + "acc_stderr": 0.010092361160364261 + }, + "boolq": { + "acc": 0.6474006116207951, + "acc_stderr": 0.008356412493562119 + }, + "arc_easy": { + "acc": 0.6712962962962963, + "acc_stderr": 0.009638903167022173, + "acc_norm": 0.6641414141414141, + "acc_norm_stderr": 0.009691180932083508 + }, + "arc_challenge": { + "acc": 0.318259385665529, + "acc_stderr": 0.013611993916971453, + "acc_norm": 0.35921501706484643, + "acc_norm_stderr": 0.01402022415583914 + }, + "sciq": { + "acc": 0.923, + "acc_stderr": 0.008434580140240646, + "acc_norm": 0.917, + "acc_norm_stderr": 0.00872852720607479 + }, + "piqa": { + "acc": 0.7431991294885746, + "acc_stderr": 0.010192864802278052, + "acc_norm": 0.7459194776931447, + "acc_norm_stderr": 0.010157271999135046 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/rankeval/perplexity25_3_lm-eval_global_step80108_2023-05-13-09-53-07_3shots_backup.json b/perplexity25/evaluation/rankeval/perplexity25_3_lm-eval_global_step80108_2023-05-13-09-53-07_3shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..b359438b1b5373f8773ccb39c348ad21032b994f --- /dev/null +++ b/perplexity25/evaluation/rankeval/perplexity25_3_lm-eval_global_step80108_2023-05-13-09-53-07_3shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.324, + "acc_stderr": 0.014806864733738859 + }, + "anli_r2": { + "acc": 0.322, + "acc_stderr": 0.014782913600996676 + }, + "anli_r3": { + "acc": 0.33166666666666667, + "acc_stderr": 0.013596836729485163 + }, + "cb": { + "acc": 0.5, + "acc_stderr": 0.06741998624632421, + "f1": 0.32868937048503616 + }, + "copa": { + "acc": 0.81, + "acc_stderr": 0.03942772444036623 + }, + "hellaswag": { + "acc": 0.4817765385381398, + "acc_stderr": 0.00498646615169878, + "acc_norm": 0.6341366261700856, + "acc_norm_stderr": 0.004806870285747301 + }, + "rte": { + "acc": 0.5451263537906137, + "acc_stderr": 0.029973636495415255 + }, + "winogrande": { + "acc": 0.6290449881610103, + "acc_stderr": 0.01357639990223157 + }, + "storycloze_2016": { + "acc": 0.7439871726349546, + "acc_stderr": 0.010092361160364261 + }, + "boolq": { + "acc": 0.6474006116207951, + "acc_stderr": 0.008356412493562119 + }, + "arc_easy": { + "acc": 0.6712962962962963, + "acc_stderr": 0.009638903167022173, + "acc_norm": 0.6641414141414141, + "acc_norm_stderr": 0.009691180932083508 + }, + "arc_challenge": { + "acc": 0.318259385665529, + "acc_stderr": 0.013611993916971453, + "acc_norm": 0.35921501706484643, + "acc_norm_stderr": 0.01402022415583914 + }, + "sciq": { + "acc": 0.923, + "acc_stderr": 0.008434580140240646, + "acc_norm": 0.917, + "acc_norm_stderr": 0.00872852720607479 + }, + "piqa": { + "acc": 0.7431991294885746, + "acc_stderr": 0.010192864802278052, + "acc_norm": 0.7459194776931447, + "acc_norm_stderr": 0.010157271999135046 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/rankeval/perplexity25_4.json b/perplexity25/evaluation/rankeval/perplexity25_4.json new file mode 100644 index 0000000000000000000000000000000000000000..5ff1be98b4ab064ad9adbf2fd6fa481a9810e05b --- /dev/null +++ b/perplexity25/evaluation/rankeval/perplexity25_4.json @@ -0,0 +1,73 @@ +{ + "results": { + "anli_r1": { + "acc": 0.335, + "acc_stderr": 0.014933117490932575 + }, + "anli_r2": { + "acc": 0.326, + "acc_stderr": 0.014830507204541042 + }, + "anli_r3": { + "acc": 0.33666666666666667, + "acc_stderr": 0.013647602942406401 + }, + "cb": { + "acc": 0.4642857142857143, + "acc_stderr": 0.06724777654937658, + "f1": 0.3026891807379612 + }, + "copa": { + "acc": 0.83, + "acc_stderr": 0.037752516806863715 + }, + "hellaswag": { + "acc": 0.4787890858394742, + "acc_stderr": 0.004985289555586538, + "acc_norm": 0.6365265883290181, + "acc_norm_stderr": 0.004800164434233263 + }, + "rte": { + "acc": 0.5342960288808665, + "acc_stderr": 0.030025579819366426 + }, + "winogrande": { + "acc": 0.6179952644041041, + "acc_stderr": 0.013655578215970418 + }, + "storycloze_2016": { + "acc": 0.7471940138963121, + "acc_stderr": 0.010050543909878572 + }, + "boolq": { + "acc": 0.652599388379205, + "acc_stderr": 0.00832781675259947 + }, + "arc_easy": { + "acc": 0.6788720538720538, + "acc_stderr": 0.009580787536986797, + "acc_norm": 0.6590909090909091, + "acc_norm_stderr": 0.009726579593424019 + }, + "arc_challenge": { + "acc": 0.32764505119453924, + "acc_stderr": 0.013715847940719346, + "acc_norm": 0.3728668941979522, + "acc_norm_stderr": 0.014131176760131163 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/rankeval/perplexity25_4_lm-eval_global_step80108_2023-05-13-09-53-07_4shots_backup.json b/perplexity25/evaluation/rankeval/perplexity25_4_lm-eval_global_step80108_2023-05-13-09-53-07_4shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..5ff1be98b4ab064ad9adbf2fd6fa481a9810e05b --- /dev/null +++ b/perplexity25/evaluation/rankeval/perplexity25_4_lm-eval_global_step80108_2023-05-13-09-53-07_4shots_backup.json @@ -0,0 +1,73 @@ +{ + "results": { + "anli_r1": { + "acc": 0.335, + "acc_stderr": 0.014933117490932575 + }, + "anli_r2": { + "acc": 0.326, + "acc_stderr": 0.014830507204541042 + }, + "anli_r3": { + "acc": 0.33666666666666667, + "acc_stderr": 0.013647602942406401 + }, + "cb": { + "acc": 0.4642857142857143, + "acc_stderr": 0.06724777654937658, + "f1": 0.3026891807379612 + }, + "copa": { + "acc": 0.83, + "acc_stderr": 0.037752516806863715 + }, + "hellaswag": { + "acc": 0.4787890858394742, + "acc_stderr": 0.004985289555586538, + "acc_norm": 0.6365265883290181, + "acc_norm_stderr": 0.004800164434233263 + }, + "rte": { + "acc": 0.5342960288808665, + "acc_stderr": 0.030025579819366426 + }, + "winogrande": { + "acc": 0.6179952644041041, + "acc_stderr": 0.013655578215970418 + }, + "storycloze_2016": { + "acc": 0.7471940138963121, + "acc_stderr": 0.010050543909878572 + }, + "boolq": { + "acc": 0.652599388379205, + "acc_stderr": 0.00832781675259947 + }, + "arc_easy": { + "acc": 0.6788720538720538, + "acc_stderr": 0.009580787536986797, + "acc_norm": 0.6590909090909091, + "acc_norm_stderr": 0.009726579593424019 + }, + "arc_challenge": { + "acc": 0.32764505119453924, + "acc_stderr": 0.013715847940719346, + "acc_norm": 0.3728668941979522, + "acc_norm_stderr": 0.014131176760131163 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/rankeval/perplexity25_5.json b/perplexity25/evaluation/rankeval/perplexity25_5.json new file mode 100644 index 0000000000000000000000000000000000000000..c5c471d8e80fb23f510923951f9c1e4b66eda729 --- /dev/null +++ b/perplexity25/evaluation/rankeval/perplexity25_5.json @@ -0,0 +1,54 @@ +{ + "results": { + "anli_r1": { + "acc": 0.342, + "acc_stderr": 0.01500870618212173 + }, + "anli_r2": { + "acc": 0.34, + "acc_stderr": 0.014987482264363937 + }, + "anli_r3": { + "acc": 0.335, + "acc_stderr": 0.013630871843821479 + }, + "cb": { + "acc": 0.48214285714285715, + "acc_stderr": 0.06737697508644648, + "f1": 0.31573655103066867 + }, + "copa": { + "acc": 0.83, + "acc_stderr": 0.03775251680686371 + }, + "hellaswag": { + "acc": 0.4805815574586736, + "acc_stderr": 0.00498601693867853, + "acc_norm": 0.6378211511651065, + "acc_norm_stderr": 0.004796478664403837 + }, + "rte": { + "acc": 0.5487364620938628, + "acc_stderr": 0.029953149241808946 + }, + "winogrande": { + "acc": 0.6503551696921863, + "acc_stderr": 0.013402073680850508 + }, + "storycloze_2016": { + "acc": 0.7493319080705505, + "acc_stderr": 0.010022263975606228 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0 + } +} \ No newline at end of file diff --git a/perplexity25/evaluation/rankeval/perplexity25_5_lm-eval_global_step80108_2023-05-13-09-53-07_5shots_backup.json b/perplexity25/evaluation/rankeval/perplexity25_5_lm-eval_global_step80108_2023-05-13-09-53-07_5shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..c5c471d8e80fb23f510923951f9c1e4b66eda729 --- /dev/null +++ b/perplexity25/evaluation/rankeval/perplexity25_5_lm-eval_global_step80108_2023-05-13-09-53-07_5shots_backup.json @@ -0,0 +1,54 @@ +{ + "results": { + "anli_r1": { + "acc": 0.342, + "acc_stderr": 0.01500870618212173 + }, + "anli_r2": { + "acc": 0.34, + "acc_stderr": 0.014987482264363937 + }, + "anli_r3": { + "acc": 0.335, + "acc_stderr": 0.013630871843821479 + }, + "cb": { + "acc": 0.48214285714285715, + "acc_stderr": 0.06737697508644648, + "f1": 0.31573655103066867 + }, + "copa": { + "acc": 0.83, + "acc_stderr": 0.03775251680686371 + }, + "hellaswag": { + "acc": 0.4805815574586736, + "acc_stderr": 0.00498601693867853, + "acc_norm": 0.6378211511651065, + "acc_norm_stderr": 0.004796478664403837 + }, + "rte": { + "acc": 0.5487364620938628, + "acc_stderr": 0.029953149241808946 + }, + "winogrande": { + "acc": 0.6503551696921863, + "acc_stderr": 0.013402073680850508 + }, + "storycloze_2016": { + "acc": 0.7493319080705505, + "acc_stderr": 0.010022263975606228 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0 + } +} \ No newline at end of file diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..636feba69053c89ea9a2a32b4ea80d66d6d4ef59 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59d23f427c1cbffaaa7fdc71092b7415f6f8dbf390b4cf0a8586eba6afd1d84b +size 199058647 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc9d07a127ca266d4086b2be0e04405311284a5f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:526c2e91017efe4b249eee008d0ed8a9eb70a38dc13c5462b6af7db9ff89f62a +size 199058647 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e57abd293197e3ea2e0b78fae8667e2219ca3d4f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb5943191b83807fe8921219c9b5da3b0f657400bb770cb355ce5e5c78e8a5b +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47d2704995e9b5ae338d88360f8ead12e37dbe1b --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7ec700735481c9275da2f051336476431e87d79559a53e5f1a1ba1111269dd6 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86945449aafe47949674c545f057246d060ea0c4 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2ae076b59f45d9ad3b9876069ac560c873c69841e415a487ad01ffb45e457c8 +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3fdadd96610d7a8728a334a7ae6f2469ed1dc10 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b1da2fc4ac77e225d66b6f9f2cca80cb17965ca6a627755a4e0430d654aa6dd +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03f130ef8a653ba7018c0a9fbcd55bafc2e2a445 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd3936cd9e8ccf10bdcb52bb8453255756abb7bf5c6d34f83bfc69fe0bab5182 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de0d7df7018e3104844df102feef728094d7c5a1 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41571507fc5871081b5ca41e176639687ec6d42eeb6793c8297cd6d966680256 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a520bec449545e1c18ffcee2046b7893de04af3f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f149366f28d5f725b677cf81872b578f6485ebc95582ca0e45d94dc14711dd77 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c72ceb875669b4b2db6501d24c4406ab2d9ffc6 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52db56de0ed4386476d3e6e2d3fa474fb1508c1f4907705c10a3381d407ff19c +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2967f8e5be18a14487aa3c0b9b125c14f50403c8 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51275a9a907905243c10aa5f0f15ab7a12c539d1c4bb3fa0d3f7886ea6660ae3 +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03caac3c53f0a0d04953b415d5e59ebabd9f27d5 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ea96888eddd5e6823df1f80ae8cfce9f86f40c5ff0f9d7e5bac74120128dd7a +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe309dc1b0d1dc80b8ca728a366e2bc120f1c07e --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15c028635b7cdf6ffdb93188633c62b951e0c3f693ce8ff125da6a40f931c488 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e87989653548d479c7db9656738e52c2c8e723f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29ba173dfbd0e1b9b1be3d3760420be0f64e60d0fb6241e7c4f0c89e66ef291e +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b262cddf237580ec347ab436a9ea8250832257a2 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ce324b273ad01e7c84ad42c9610f3b28bf0d1b7c0c149608ee928f1f1acfea2 +size 199058669 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d16883c0434fd3a71970829707b87c991e7ccdc8 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:404fbcb7f00522a17320f9f2dae504dafd0d5f6d8e478166ae68628941c6b64f +size 199058669 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bbec111d3ed7c81022ab02a1ac6c6c39073823b0 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbf70b0f78807b61b84bb59be0fd1534b187e3b01efa0fa72fc806d758259b53 +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1abe35383ca1aa5546cd3e59fab1dd0bbfa506ee --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc7d3c10ccd60d783e63f78c6e16bb2d6368d3d759f30cf62d5fe6b52bf29d20 +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd9cf7f987cead41d7008f949807b27454d36007 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:062ceabe2a45af096e792eedd81b9e289cf781d08c454aace4e672d0fc6dedc8 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd46557fb2eec2434e2809427c4de43e08a3a638 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:773ae6aa90a3a0f4dff9e33a1c341b1bfc457587ce99ae67ec825cff399a3674 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ab9d849ef6b74d681f8221eeddc71ba72fbbd07 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0a235c6dc30430831d1dc7fab87b933ad30b76e9a6b2755aa4c2bb6558b159f +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b900cbaba0f9aa29184530cd43f8acadfb085c1e --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6027b0076148f138a5b7e72f173c9e80937f1cf064108dc021572d5c94237581 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..685c533325a965ec7f2a6cd2bda3492afedc3ec3 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b0427c7fa3653f697f7c6373a7fcb232938ecf033008d5681cc32668d073eee +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a31573696db629bb5bea5a44938f0de7ee3f84b7 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fdb867ba5cef6bc61db5ee9a9c15b21a8acee7d3bc3d3aece07a174a8944c7e +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f3a670f46139059441fcb72c342cf6195669f91f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cab7f776934c66398155f23223604df9365145c0305151c5779eea6fb410573 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fa50aa0f8ae185ac30c091a35182e2241b4dd1d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8af61c15b47949d93c7a82a330038d2041435f04242491df9ddf3b506c20c598 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c6a71b1679ddbd728faf2d8f0f74fef2ca5a90b --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b4ab703b0002c8335b90fa54a8163364681dc930e0ba8945abe82e3422ac324 +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..96f3e3b57d8005dd20d60e91c7f366d59c6624d8 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7581905ccc041e121ceea9ea46a1fab76d42c7e5555f72412085a104a7ebbd3b +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..15d31ffc960839997b8752d9f040247ca849812c --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:371d6cf07edf81d946605d7504b8c0e14323686adcacb9afe26a4edf7ff7ce91 +size 199058605 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85b800deb6961d00038b6e0819826d3e1d3dd7be --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:644c7e7f5f81e9660ccd7f2ff411c2ec9cf27dd4a4e594f4644957c8c1da5d53 +size 199058605 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..afc6965053b8d719937235c188609b3e58b18198 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:598e6960236fff159e2e2c73e76d07d562ec67d4a5b2ecaf54da149dc83ec5cc +size 199058669 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d1a9667f81703390035ed50702c21d83b4b9936 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d8059949fae7712781495ff018f58577a2f2c3677913dfff3f235327eba5d47 +size 199058669 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a41df33e9f979ead90774f8af53a04c4f90c3f80 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8b5d760dd7ddd56d81ba3e81f523a4f790cd35101b6ef3b1c647bcb8e07883c +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c3f57f3d752e7e245de6148ffe4f90901d542ac --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df500b5ad92fe305d65c4058399c3831969db8a5044fbe4dc01f65566888e920 +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..72c30160401bc52c0bb170efae2aed8c29b32b73 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25c86c3f2ca75f35bbcac1abda0115050c5c7865f7b428777fb16c9ba22878d5 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..772095d0a701da416f60484a5a5a7c6dbad8e7c5 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5472bffd78e2d7291169377e1121a9a60d951f78100b2dae0ddbd2d19134f50 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99ee0f3ed6f4c36ee1f4b0fd069eb533f9ab6f06 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbcf9c7b5a5cd375ad3a8f555550e65646c8b47cd044989507edfd2219f4c1c5 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b6e9f9300dbdc9828346fe4552d781d3c80debe --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2dc0736da3656938106fe0baff81e4e35cef19a5c82d09e247df083d7a20831 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e75b407804c0dfe9225d6536e0dbc2edb2f52abf --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c7915c994df564e0449575814a52c5e0f90819582abf46a53c12d1cf58ea2c3 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d60509b8e4717b0386d4c0363231acf00bf315a --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41a83a119adc15be0e81635f899bb90b802bcf857a87f1e57be84f7afe8a3815 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5b47f96114bf72e06a414f1857007538f119b3d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a079d73ce0151886db5e24dabae57a87d78de445cda632bd68c3e49479ea6e3 +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..771ecb127df4fa5f9f887ee13a36cf075ca6df01 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee0f11b207f3a339c278d8a948b582017dffaece915711719745cf6db6801d0 +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..593644fee9ea6aab6d2921b0a7e0f7b062cf9c8a --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71b0542844167cbde9ee08d24f2c9c06e1549478addee17ca1ed953b421a1df0 +size 199058669 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..793a068f41177ff98f4a4532028772197c96cf09 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58d0a454b6ff465c9e865d4b393e79d1297cac40164e8182173fcf5741f58d36 +size 199058669 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc4017af782239751dcbc053fdaa2477576d0dd0 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee35bcdd63754b5e2274300cc12ad02e7fc6c5df6b935acd49534950245144c +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2edaf67e6bdc42572d32c1e967b52a845f4a9c09 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:556f508a95a0e9ecd051fc1576115181f4a7ce9ab99a4c8b994deaf4f958dc92 +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ed115628cdd015858e13b3c5c25fd007267da6a --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64d40848355eafc99a693f1c4f4a91808486c623ebbcb324770d9f8ac9b48426 +size 199058669 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..13c382165718ffa48c72f8bb461451e5e5a253c6 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34e193979247b908db01f87908599e98ea6bb3f8fc696f1f83b5dc977059be6f +size 199058669 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1964310e7907915e486bca276b309a49bb7c03c --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b30f316370fff25943b2b08095351c20ade72ce83cc41283dc469492cc99457 +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62432a4c8d2c5b99e5597f492af3f72aeb3d53ae --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ef192ab9cc4e3973a01469a8ea193cda0426cad19f3f0e861a1508cc809d4c8 +size 199058797 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ccb77e195b0e606b12e79316d6217f2303613fe7 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f790498b8d722fd6c0e20586786b5e7c15b949aa7701b2e64a4e7a4d7834bfcf +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00d38b14dcb4d7e18db3bb7437253ff955649814 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bfb6aaca1e1390f8678cd86abd8dbedca57fdbd712f3fa4c6ee413b3f069010 +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..415d51dc69f4b644eb76cc4feaa39b144d2e850c --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d3f0b65e290660452eeac12e72732d83a7932ff8665337428e6178e66e840bd +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8a65000d43c8d5da4e4e12f8addc4e0072581c7 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c86162119d77efdb89614048dae7be35dcb2b6bdaa6da5f6763ab6f85320f12d +size 199058733 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..71d8aa45a58e97a4c3e28bd2d41a1d86d2b97569 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3b830ed9111c084d994b02adec9c448bd40e8d29914ab3ca75b2cbee6720680 +size 199058669 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..358e583aa437635b544ed344a977689ef543913d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c77c6ec1becf6d54bc8d8ab049eb095b30fce74cfdeb4537c89b1dffe8c2c81a +size 199058669 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..25bc01c64091594d99a1c73be672ca58213aaf8f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ab590fba77fbb7ff1f1af9b9d79d866228a1877add10db11368f8733d8f48fd +size 199058925 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b447d7d9d89322ec5ae51a7d698b46397714a2d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d7f08f35ccc89fbe20c7bd30b818bebb02a8a258dee2faa9ce8ba80788c2f75 +size 199058925 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7868f9f109a688c5e307b8676b6cdd400f0b3d11 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:483b1a275ede576035fa9b593875609180fa750456e8a40320468f7613ddcf8e +size 199058605 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..26dad0fc91b57fbb3c28d86971ec6d18fcbd65a4 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5127765df3fe1d8b5dc8d62848064806b209733d9fcae6d353f669eed0212dda +size 199058605 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..315a102cf90e3fc224beaccc3977af4ca83f49eb --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:193e39659ce4b9c5a23520701bed283ddb12bec34e533e691d55e2e065c93260 +size 199058605 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ba6a1ef10a95403c98d8e59d08da3aef0b1a2f7 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7bc1e8c50ff6e7a6ab8c15967b9dfcc95b8d5c8e5c57c3f50566e175ed391c7 +size 199058605 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..867ad4943f6a3f7c3879ead93146f1975d3b6c59 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f53af657cff2d5a18c915d190dc51f61e9ee0228f78829494ce115284814f4a +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..656bcccc0f2b9731ae556956721ea1a002979a44 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2011076766af843d25358d3e761a93ad4694aa65225e07801afa7f3a1db96876 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb7c6036fa43e3f986aa83b8699c86ec98c656dd --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0c2477a4a9c74fbc34929cb8c83a246f78bbc4bc2ab929adc229fb602157223 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83d7991461336d9702b48aa445826e37d61a435c --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0769a330a031387bb0b4d2ae6b90d6cb8e38c08af76bf36d98206e0333bfa73f +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ec5b531b7f2bca034c1641ac39c2813b1837521 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6906fff8503c452018cdcc2064b8bfec126f35427fabeeab1438a0699389d835 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b74eb6a7a3a4c8c3519a532f51bd54ff968bac8b --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3911dd0ffaab45464bf90bc79ce67a0d4c2a1f90e52dfcf1843c7e9ed572ff39 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b03dc9571d0ec09863dad2303fc13a70a9b6d27 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f272996792048973d64a202bd4731dcd4bbdd9aea51a94da6329c27e8128eb8 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2bdd5310fccd5660df3a336717038cca3da04fa --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9b6945cbf3df16f9b9d785f9359e7225c0c36a7d1e1aa047ae1b8fa03667219 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06bc58554230ba599734682441af27c3a22cc749 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cffa4ac6117f1593eab978ce9e4b332e28a97e3632239eeed26a41394654018 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0590003161862a2fbeaffb2b968cce46d9658fc --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c316eb146e6f74b020bbfb55621e959d4f3496205054481270e436cc4fd8bbd2 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f0f8a7fff562f7b2cf69441661f0ebc8c12d56f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3f3fbb4206b6dde97a28f74caea4fcbb1643935e466689e9bc3ab3d7a61608e +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..119111770e4da96682ba2af5eb0aea4d9922ce45 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e006495e448c54a2d6adbfa977a7530777544ed6bbc5db31c0f9e2db76ddc06c +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7767d0f8a96dfb3f0ffaec3a0c290f6ebadc9210 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a30c483cc345adaa13056e43c2b9c5730aa9c8b0ed3209eb4d37de0f8552a17b +size 199058978 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d9c6ae5efde4ffe03c9f090a8f5fa3e0a9d9680 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4466824a8b2368d7874b68de08f722b4b3165d8da336943847273b774214068d +size 199058978 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb85f963cda5c34c9733d12e2230c88eeffd8ba1 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9beb1ede32af3937f5e2b7e369126bb107d2780a94235350509667389cf9f7ad +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b384d9e44a5f72c2a02ed0db1a2fe1672b3f5d2 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e117c25d4e112c5070e23274db09491a7227122ad9d7415014539ef8f7b50812 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d567d84c5d6dfdbc9c3bb217e0513f2419cd31d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:423ccfb5c22f9c9a443e8525cb5354725ca07ca2c108f48901b153e0b7979b02 +size 199058647 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4dbc9c765653fefdec8c8e86b6a14913046475b --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0263caf6e79ef507b79b5a9fd4c854083d8d5ec474e55c63e9574f709b93bb2b +size 199058647 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3830c148c1ee7ebc53b70583e88475c01a70926 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:260c1931e3d8c263c54b1ed0623ba086c005b5870dda5087b2fc3af03c907b5c +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2fb912d9ae20b9bc9c974eb0b5a15d9142e687a --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b071eb8e95753d85e0c1986b355393c57bad2612053bd044b7ec8a8db14a12b +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bfffacfa77e6a76f3f5486e4f137303917694bc --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a98a9c8775e07041ad8f18dffe301d5a39cb3b0f9f22c265daa8416d41444ec +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6cca54f4ceb6092af7ab82ff3d8015d14e8ad1a4 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8af40692673ae308d96c0b9e61a4e1a0176ac815c0542747f1ef35e358391426 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd8ae128f8b66b585ea381cd2629eba06dc92e34 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ca08f17bad919fc0c05c96e8dfb2e54ada541c17fc16da5c4418bbe7388398b +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83fc87591640df994bb4712a6c5577f4046e5314 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ffa85a3443d04b5b8ae9056f3e6134133fef7e78b073c1dd1c0239df3f4f8dd +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c073d283c6d8e1b245ab409b66d3411c81526764 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3df30a62da6f483991f82ad6a7f030cc76e05b36e3396a823efb9867990c199d +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..764460b292c81ee18d465e698424d0ae99fa7139 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:905ecc8f7e07141e57ddaa3c8ab8f87191d58a61e6ad2a8d13c12ca0b53516ab +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..da58b1add9ac4a12d44bf694d608a0dc5fd8b767 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0459234ed9856aff9fb09fab366e2ada63a01250fbb85e6410ba49caede1df6f +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ddd2f57ef6fdea0a2d93e534afa9230b64996c0b --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e01f3170f1171de5f1a5bb4c795040a1378e944c30518762efe23228549fa9da +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6521ad669d1825d72cc23078db66d7e59ae66c54 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e4cc3349079ec6083fbea7ba75ef79e2432e92bd38a52c1b5ac96a013f33d5a +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03551a72f683d4809672c889e955fa2255399385 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad32e7f14dc9a8805147a8836c68852071a4ffba101fa46f586522506637f220 +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d906a8cfb725471fa2d9512cf5ab08e6692c4b0 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a5f844f5e0bae15f9b3d91ac009f4b07ebdcdba1e72c9480fd3e239f35e791c +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0a558d6c7b294db67e1b7ccd34d1d2d2d018f9e --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c052f76ef900eff27c32fb812a09da3ad04019021f4d679d00b82bc69f245cf2 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f02e01dc6993f39874e9266ca14be5a31d5e826f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d881f18f0543e5352bfab455f4a6a6a2f1b8f0307507fa3436f2f73387e4b60 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0870fbe0c37032fd181f183ffb9322f759124239 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aafc8d1c10e8109e2666cf5cc69d1beefb66f63a061b3dc9442363764b0bbfc8 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1bd8c183659c2f42de6112f2763e1b8bea72eb5e --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:384f0640b39aab68cfe2edb67f3d94edc5b11a7743cda9a0a6bd4bcfe55803fc +size 199058594 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eebf3f2df9c76cf929907d5d48ff7d53f5fb95d0 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:024049a747093120e9e5d2714b38c713cce22536031a8fb51d9f5ecf307e7072 +size 199058594 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24de52fb50bd87a5fc741669dbdc06f87a5fe0b2 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc6af5b38558e84b648ac9f471d1b7d726a295670e13e32ffee690dc80bcf2ac +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..582ae45759b0d6a4f7135da62b1378df963da6c6 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:362a018d713a26d3a8068a8896e270c0bbcce0445bee3f9345f236e0999350c6 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e6a5d51954d4e82f9b0635626c6001af3c18a51 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ba881b3e8ecde68d6e91cf47efc197234437c47d32e4945dc3d4c7ff051d3a1 +size 199058711 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c60350dca03edbe27703aedf48d965b8a732bdd --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbdf2f9dc5000ca893cbea9cc8de9262237da5cbe98cfa25bd1bbe0f3bc5d966 +size 199058711 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd1cf33a941f6b6f0aab2653cfac3ec80c33e202 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c7abfcea971ec4c3d2c36e0429c18630ef4f3e63799589d1ca8be4f56b918ec +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06d678a627b2fd996876f336f0c1f185b4b8082d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f2bd3cbb718cc12b0b269d5abd211b1beb0b3a5ba3773e04b2b3804e58fc377 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a943510f7af59fd5fdc718e49d202dc83ed6d4c9 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:203914c2e05ad384e11df0cc45b3ad94a61493965733b18194545efc6fa10f31 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a1b25769324c32c7aeb969fcf30808725395896 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b51229545ab29a600f9b524e7f097221b32c53684e68670e3f1d10a12562915f +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..07432d7138ab846c925707bc7694f619d784dc89 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:318be5ec09619c6eff54cb38e2dc204e8b46f28067813627eed2670ed1637dbc +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91ff079b0ddc466b9ac06a5a496127a537783b63 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a35f9fd034a1773766d95d48659bb23fd5fc126fbe5f4cf690c25260ab223483 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fca7e2a55a7154b7f5dde3d263b57fcbd40dba9 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:056fd4d04ab2db9c416034ff79996e240908202161fa5da7631c0b321150e5f4 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f2365045bc1060229db63500e8f560670375703 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d0e494bc813f71f67c4df4eec9cc26b5368682a18a10a8dac497deaf70eb29f +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ec51cd68979f60807e4043e9228390306e9d31a --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fd6e0529f819b76aceafcce015d6c97ba51d1224f88064757591324cbb0b7ce +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0ac03de1062bf1e3c77f3a6fa84ffb607fb18a8 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c682c47970916396afda5b3e6343ced91522976e4a210784a20a2a2cdd807478 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ebc26210c047bb44e02c7ccd0253f7ea8f40437 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53889409010fec43661fa94e4ca316f2c4fd0fc826912d44cf3c566372b4e7ca +size 199058594 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b47b5075df7040c6636807d53453d069f4f63a32 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3f6f16b63d305cc8f7f52b17536f97ce8a89f4b9275f98e2b9b0d8c222ed167 +size 199058594 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6f002cdb41b014fd38b908a94d84c830b610ef1 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e0fa0764437cf96687d1cd74bc046fc638c9a7d5e58894f19066d9ec522e7ec +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d52818c33e77718fb134641aa36f598e984df2f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6cd49bef33e28754610dc0f90e381ed94813ca9e16b92093872800f56dbc2f9 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e72ea8d5370bb8a9b6d52392e4fe8ae436aa9089 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f683ea2e009bd0c61b3457bb7395b3dc86e8f098e41376f00e43b4e8b13ed337 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe392c6de94c1c937d007bd277c4e3e42767bb35 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b442b6e0980122db789f37297fe97e488e89b586ef5869649d37d3be77317981 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..403d7495ffbd72a06bc0d2054051387a56dc03f4 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93c194873d05420f26827c4aa0679dc649a8a06d5b5c10431bae3e06546d8600 +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a66d86c350dcbe530fc2dd2ffa5604cfc1b6ba2 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09a0f4b85469165d320e7eb2070b6e4c92559d1eaf671f4f95c2416ad10be85b +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dbcc4341a2c309f87ad67bd1f0e8bf70d1993a1e --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8ef3efd47257e12d1edfdeb863d468c3b2c8cd62deec29a5d9f39e877d32ccf +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..90297dc8d6666349d44c3f5998619244622f9c4c --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9430f8030d9114829a7c5a7f8e83ddff65cba7fcdc3eec1fe8121b3fa67ca699 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e4d78da16bfa05936b92a8a800c3acf379f65a4 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab5c0bc7dd7963cc11bb21aee33392fc8ba5f27892f456d5165f3bf65ca6de84 +size 199058647 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7804662a27d67cccb477bda4dbdb09a74f5005f1 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d02c444158cfff8398ef52a7d580ee8fb66347d325e4c6866aa2027cd7e21809 +size 199058647 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3cddcdb8cc30ef69cbc5830f5f8f11466a722395 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:100922a678567c82723f8d242aa08c48915a5dfdc5d7b15aa5f50b59afa8b3ed +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfafa0f1ca241e868f51ddc27852ef1cbfe3aefb --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4613ce7c9d829f7e0b7a2cc42c2fd535e6e997f90b888cfbed5e2ce22b69bc7c +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61c24a2e4f019ef1d8c42bde7e2702f4b8f74011 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b031bd6f5d6c4dfb09bae4321592021e09f2645db0bb7587c1f920831cb822a6 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c5a8e369c4fa4e0860fc1c866461bec1fe08a87 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2218ac7b7dc185665eb5f5ecf03239ebe73ebfaa3fb1377e32deba4360f565ad +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..efddb85e257425c5178e08063767276908f59f8f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe3a12c7c40909fa726bfce4e0bb5a901839947f4faa66c0fcfd234ba09bc3bb +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4595615a1b33438987ee30ba6a8e3aa89647b046 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:961ff43019803c1310891bf746698f58fae7bc9867bbfa0025ee36b7fc67073f +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c89b3be553c29430db4366574b2ae6501039356 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21a540f5dd96f5033e5f05992780bc73adee922ef2370d40819689fd5cbce66f +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0fea8468e95c45e5a0acf3654f163243aad7556d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f150802d54f238468f14355e4e1c8cb3b39f1bfc329bee670a398327ad6aef0 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c5eafebfa5b59821403c8abf948ae75eaadbae6 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a96be9c6ebacdcd1fd4bfe98c52b4782602b2335a306faf655e17633435320c6 +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32143c622c137224168390504eec2d59531ac79b --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adab2f2addb60c33dbe7b7898503ae7527bbb7c26dfb47b862aa03284eba2616 +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86ba43e446cf1c6f149b3ce6b7cee0ff2e1a0a79 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:614eca45f7f9214f2ec41c53dfda32e1a217250c87eb96d5ebabd5ca15542059 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00c9b66453649267adf75dfcc23887627b16fc83 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7967f3103316a365a4236d8b42fd15b59a627403b73d148c9d535f968d5feef +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bbea4e38f58404aca953fd51dca75d9f785549e --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6af74c9ce3d66b1d3c5d4a723d7be5b4d9c5aed20a54992f148c4db5bae6246 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f44fd78bcb46779efbb4d5a4d2cbb09cbbbb4bc2 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acc322e5d530662b738da291f541d4a358f32761859bd41e6dd8e43bc797597a +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a25d3bb7d4acc98fbea3860916e8a4f030b863f4 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6d1c3eda9fa81702a7809e584d345641ef27e2997c7a726ba35b2f25cf6b3e7 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a0aaac0187d8c3e2c6480b9046c420fd8d4df52 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e072f82ee379d35867d9d2f595f1ed876aa3611f636498d410182dfa42cff70d +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fcee6009cd6ae0bd10adf594aef13817f01a53f6 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b192a438e08a4772f375cd35333148674ee0ae72cf6980f788cf53a51623f8dc +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec7a334dd9e356061ccd8888fbafb82b3aafb28b --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1961c578b943015cbbc86b2f9282bfa3ad962853efaf25880fd90a78f11e8ae5 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..79d83e1d28158a2d1b00fa550e465e40d4eef2b2 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac74061ea691a16a39430311a889c154484804205317ea8557eabe95ee1ea77f +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bcee872e34462fd800a6b1928d4ff1f7d07a67ac --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a93aed6f5401e987966c7d8194c12a41b353ee0cd317bc6760c7cc0391d0faac +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c403ae78e476e0dc7083e6081cf9e3652fa242d8 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ed8fdfbe5ca0d26abdd93c0dd37511c940af2115d83e01b9d7d83be69790199 +size 199058775 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1f33a0872e655d7f4449105f19588f13de29a66 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4eef1b05cd41bed7a3af2a83bbe37f0d0bf0e42dea07d9f186ea4e9992fe116 +size 199058775 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bef603b1d765185aa464b750f7dd8d8b8e496fd5 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a129a8f471c197cfc01457848d4fc53cf6144be70919b00d90cff1a93d4e39a4 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56d0d0d5b4dcd6c6c01ccb17ac952f49efb8478a --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c99cae5b457dd958ec0c1aa9d9b5ba1a98e269837eb2e212f71f8818613529ac +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a30a95fbe33cb30a4017a4708962711d85ecf3b --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7fac1e01d01149b7d0c109c5ce2beef12f860cff38de29c215149bdcb78682d +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..621cc37f2a6f18f0da7a98301970b49137023e3f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d588b1bcab835f2da291063be5fa4c5352ee0d59514742874c7a4ffa18dfd25b +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9fcd4818b8a530ee73646fe24ff5830724b55a7 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd25e1ac27d5f02441c2beee3e977cc5257760b462d1afa85e1a1cf3775b6194 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c8e955a0975c4b8dc47ce323c294fc1b8705003 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dea8b10c40c46771519821b342b9da4a5067fd3d1b43a3a7852c4cf605d33faa +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c3b268e666577d7f0a296b1cc4b52f7b65cb0ee --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9426bd37e8fa62a87e2c0974b2dc9545845aee8797a81a8c53483fff36977da +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e457e7410dbabf57dc62731c2a6bbfa022e38ed9 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb7383dbf00991386f99caad96f5edf53dd26ff340c1537ea88bdac385fd2e15 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5a96b1937c854285be2bed1e1a9cefbcf903d88 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9035110150ae1608c772990b3bb182867a91583f20af5ec7248fb5535d2d43a1 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e11266a1411d9e15439962b2d2253c3988ae5522 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79d6d5d20a94490ef51782b6f0b4b687efd2c3dfbd0d81a2a00107c1b3e1dfd5 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32904bfb93d8d95a1f5f389b0ad86ebab7cc9dba --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9223d32fc6b7b9a81714bcc8141349a99902cb03906e05009b0ee5e419763934 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b4d4ea2006f6abeeaf0ab29173de871896cf61a --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:694489eddd5c7d83769ba3e95a757801ca8305c3a4622b0a6b5e13e70fd5e7b4 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d68e0774359ebaf287839528876b95496c604f2 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38cf7bdf4682c2a355eaa2caf088750c5bdd0a054c6b48a9bab7d5e74208d6a4 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0bc91d2e5b536c03b3adbc6203e49b22db507ef9 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a263fa782d9c2a01667c5420b484550716218c7121c03c957475d5e692f3b17a +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ec33127d333d13b05dbae034efce35e9947d708 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b94633436caeba65fedcdc5ae275310872c291dbfe14a693f4ffd19fcabde507 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39df39c8ea4bf29d726b51a8a12d4ea3786dcbb0 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:872aad3fd2937db0c81f8a2219ad6487b9eb36c10383d1f8c4cc8e2ed7b22186 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c17505750e40565f08b1ae24e3f942c6e1bcab3 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6d78714f2e9eede3fbe83f2a610e5615326475a9df3ef1a90d953df2067bb96 +size 199058914 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba174ed8ac98500d0dc67fc55972fcabf56a4a60 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a175e077c73f40f510737290dd567eae9647a542d4897ad2922d2fa39d7888e5 +size 199058914 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ac217d2b67395c33a2655835b9077584d041e6c --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a922679f89ac199ed274812cc9ac1411fa8f76bf91ebd0ee8eff07201633a81d +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88b55c1daf5e6108701499ad29e48015bc60153c --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ca1dc71109cc7ccd98e3bc6a9f05ea7f4e6f4e46bd56f473bac51266dcd291b +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2684913dba0379f12cdc1a7d76665a6dd468de73 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ab6d1e873ca2896ab3e71b4bdb96ea9be4b7992b906fba10904c366669bb6c1 +size 199058711 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d30febeddbdb0e0dc4f2ba91773dfd238b23f01 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cf42f63e5cd3db596939b54b457b5d3887db61289fe0b9e9563a0a598bb9551 +size 199058711 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d62a783a43492d250251271b4c1c0db599b27e2 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bca80302ff3bf6bdca8906b065d930deb464a61281398706d278d4ed26a054c +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dbc684bea29bcaa00a3d91e3999fd89d33ebdc13 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:193a99b65af506505eeba442f07932d7ab01edb6d88f341d2fffba4e6e09b197 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aaebc0e84cde3d5569b3796a1f0240a2db190769 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3f14c78d9a0ee0a4a81c135a5acfa3bed0254916574cb6fb27ffe847a0e8b33 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..488d8994a17c2df38ab5c26e019fdaa4b9307f0d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf95392c03ce500ed69dae402597b5200896c30b2cc4ed7f686b8fef04d74481 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b69780e2e993e3471c946d4a72addfe058045c0 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ff5b761a0c207a6f75053325f0f0115bfa8872077df18df7acd09f878c240b3 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c771ee93180d0e08286805920a69306f69da9b5 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cefa0ab9f555e5524015f5b464a4d5b2f6280a33764827052f5ceefdfa418c4 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0c88456f0176ff6a25bee5f3d257af34232e9a8 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d745cd0d0168f1336a49ebaf4bdf5834ab65ef985fc7368520fc30a4edeb5e4 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9eb744a92a60e58f6360ace5c35efa1b70cbb77 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d579faee0ffee9cfa51c481db21f8edc554ba5659d6da0bc2691e62e41c70f8 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c140722467ed894d8f309026bd16dc6dc679474e --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4db091b9b6c10c316339c09d1dfa0fe7908a44c06206277c8a9f5c690364a2b7 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2452d8ff9b3362ce79594535b5df9398f2b6383d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4477346f5a83928fad2e040925f2d77ba6e1b9f5cb480ed17db58527b44d04d1 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b162a7665734e1d97c0a96484e624a2a5afb41c3 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:342461890e7595d9e1d5c35e596737ac08d4ceaaa01d5289001afc094424c554 +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..364ff7d14d1734614b191c1bb74449c762f69824 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aa39651891d4d3c88b4e9b3b941d6f9efaead3fd782be3441f466a89896fed3 +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ddb32d2b1fc8d77a21c12f2e3ce2ca90b8fff47 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ed2a57ea950e9604170ef1f9381f8447c36e2bf6f12c16ecc795d9db2b4127d +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0976d30bb4fef5479285912e092ef90a522fa93d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2caac9aa823b67cf9fc0fbf74981615000827a052a7ff96b38e5904bfe296a21 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f1732197cbc7f3581caca9d1ae42bfb46ef9ea8 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7926e946b2f6275be0d8e873696fd0c96bda448029450ff461c9d083dc4c201a +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a351c6d216e18070e4453b3ccede81a33954d3ed --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce3709e693233e1af410f309bcb2f07fd84ccabf2a46f97fd9140ad2bf6ebf20 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7cd90d9d0b57723ff7d4b1fc5cd440248a197e8 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cea0fca31219dd269e7c52ad4af792d45380003c8e82d2b9e547e9c58eb497f +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7935f99174928797ff69ca0442339d800ed6c2a4 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2970520fda7ad5f99bd56053afddc6fc63bdf5e2cefc9de92e04e7f221bed5e0 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b61994dd199c87d5be317a7a5b934af675800ef --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c642111b94390aff4a16c73045b141adef2fb1f881d0e4d1f5dae69b1e181c69 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1817bdf56e7c8433ed20f8daa8bd4f357440f045 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b277eecf85d95bf0074be12ce88ff276092882c531bda1873472ddaa5177846d +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12a7b6ca080592269f754ad5eec6c184c6176fce --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c015a5b4388db090bd80c37eb2c22161f1c9cc9fd911f9564868bf701dc506ec +size 199058647 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f3b1d316e2f11306eec82878fc961960fe6a5c9e --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:622d75f1e8c7855f229dc141e151f5dd7096d9d9c68cf08ac09540b6fe847d85 +size 199058647 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94ab7c53533b6ff2eca6bee37eefeb0ecc4b97e5 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:811dc5d9c77a957e9d0cba63703be0b3823b42dd13e2289f036d8c96fea102ac +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f4d210d7ea6b82b12c17a25f53393bc4530b2fd --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cb3a7be903afccf1395492e35814413a85bbd82b75809169dd70e7d94d2e24c +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2fc24947434171fa8bf952652f52371cb7c569a --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6c1b692a415818b10791f53b39b00568dccfd797cab46136ed971fa26d3a69f +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d030fd7aa303cca91032f7556d6143c1e720898 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22a74e725a2c4737f28c762b12edc8aa7ac96e06c1ba31b41091e6e18ec9305b +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2bd01bdfb0549b334c16ce454bfadf1560a6ea76 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e82e2498bc69002b795c5bfe8b22556530cf199371cb567aa893a28067b807d3 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a92e63c9c9fdc048b8720d3c9e6c5966c7aa4d1 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:106380c1e4f4748552e8f5de97e1e8fffc11a8bf9a9667697ba5712df40e627d +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0906422213a97cfa2b5cd237ea36652a72ac127 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbf658b7d4325058f17debeac4bffecd02eedf08243b25fd65105f378c3d7829 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..344a68b6a9901b282cfa87d8a72532a1ccc46da4 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e5be1134113ca8b668c9cf23085c5885e9ea38af9a56fa5cb2e94e7e6731ce +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ac1a5b1a59f17aec7cf1f67b28f1da956da7fb8 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a4b945069e2eba6566c8d5d1fb9040de2a8e1121803ad588df83405bd975feb +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d133fd0de7f89ad4cf3a39f000fc678de01602e --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab6977e65eb0f8c1ec2c053c458ae513d154837edfa62ce2db7cb63059f4b7e8 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f66b410d6be679cab472a6b6f14daeb30643c641 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46e76e652a71b97f0a71c08a18f9ebf395fcafb1057778bab13137f6af7a0920 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cad06cf0e0ef52a994390b50f1a3faa5948c3671 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5c8ec62dfb1aa827bc5d28a0b97d1853301a47602e157c247808913d05455ca +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..01a6ab2c9a03c1720e218d27b8ea67a0f45ac09b --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b8eee333304e84edf0c6108c7a6e264f0ccc8196b832e8216f7146a2ae6f36a +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..913cd4e3ac03f45ec38e5cf462079169382a764e --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0755c3f08d404bb9d5adacb3e9e3b6b024f4f1832494a2f14ce0fa8a0f2956b3 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..64ef28b46178384ee48b4e6948ee19f70928dbf7 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b4f3d73b1f245e1baab4833326f1171f740fcabecfee49d46a2ca4ab3b0941f +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f6cbd8ba209fe9785ed29d87f5f07ff3c446445 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6dcf34be0d4744bc46fc1d8321dac6dbadab96b7b07e20e1c141ef474bf2392 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f9e0c55eaeab24e69dac1c0130e85c5c0d41988 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a849302908401a908429807974d64b1f871fade283e12a767bf9e9923e44fa4 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..088a2d13461102ddd7d97836e04f50c90cd922ea --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e3e4d325463d413af179517b13777db6cf7666806f20a7b2b286163d77fdcb9 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bee6f2a2c0382033ead86ac1aca0604476974f1f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e4d3f9b63401842cfc17a8123581e1139be4f66b2340e9924bbe75552381b8d +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4edcf0d6ea57d59a71a3e6c957dc3401b562c362 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e892bd15f6e3bb182a62f8269ac72dcc303f5c535888a4c6ff0c6d108a2b7e9 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6458dc6707f06651ead14ee7e8be45c9bba9866 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7ba57c55cb6378212d6bd578d154ca0f7a9b2b382e94dacc48e7ba5572dac9d +size 199058647 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1a50e9edf041a9e55e76e985ffd9e6b04af8e4a --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f0708d60ecae81e8a0d7917a9043902ef5119e63fb998848c13943df17002d1 +size 199058647 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c5bd56e8d7e41a23884a589a3076f8430941553 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a15e90c23cd0c2c22c7c44612f88e348671e05c86445d00f1348979511aaf282 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e84e8214585412fe4408d0677a168e496fd6e700 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:099cf88600c4e7c02cd4496132f1a4b936c9ee928de00154635f5159c5361252 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae4338ed1e32a2c0bd7ff9171cc57c8c36ef17ac --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:123bf150267e3494282f82fff5f998eeaa2bdf745ae6384bfb33a0093f94066c +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ebef3410023d5a234b6b73d8fb3dc890f402d18 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aa032c6ecb592edc7d526f5ef12117e6abba8469e2ebdefde57796e16602628 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b7015776312bd822504d790823ab894aad6ffb2 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48bc6e5d0fdb69268efcec07c6803171ffc6a91c73a98948cb013ec764635fab +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dec10c590b25931e6995d8ab2289173955463fa --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf9079d3fd0cb6e027809b2ca058b088c028e5ddf265c7598f547dc25fa88430 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d90be2ea134ea4ccb6854783d44daf9bcc01b83d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bde8a14f4f97f314b82aaf32fe69ff84a0fb3849ea0dfceb774b7b9bd4bf85e +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e5ad50304c0c8e7dd813e581dc0bc14d452ebe5 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:863177cae299f7dd0266eac4d31babae7ee3791ff44b9a00db854871a8d3551a +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b939548ea17488057846030537f57b46e6898074 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8556e6d1d2f624f1636be7957c92537804c4abb06c050aa50d5f04f726a73eb +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..449b5dd491aa1e91368292cdc060267a863ea2a9 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bfa8c23cda985950063e3fe1950e51c493c78ace6e002ac1f9669c21c98f56d +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91ed068603ae91e4a1b936145fe2144952569629 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de8bb4b48554b1f5a6637837fd707f6b769e13a3ad5e8ee0f6595a57c5a047bd +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7992806f3f2a0d328fd67a57487c29480e530438 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dc3a7cd467987eb8a883fe83dbdc83e07c0e960ac3062db7785d0aa26d59dee +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5946fef496c5f796cc2b942290703ec18234671 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66ac5cc9b6c974eadb056540191c523a07c97197e95aac2ff54a0f402e6367a0 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e216a75cf85b784651c3c8077a88dd22d0239cd0 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce28fb970f7bb50aa0b6f0be4e48705701cef3db55f14b97b42e0f1636d9078e +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c14c6566486c66f776e9242afd94ce6ab0d94279 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad95195c26b707575b18cc841ed8d4048c51f9e59e3be4c35ec08abb9438ae2b +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2482b17bd873610d8fc57c3205f4033269f6b606 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:193587d5fba597e2c26b9ab722951ea6a1ff2b59149f925d6f770034a5b8a9d0 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5dd9f91271bd846d2b2668af8643b57fc812a559 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92aefb0bbe554c6f1b92b8da06849225ee44419274dfac4844af35e6948ca117 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c09699bba0eca0376275af7351965eeb9dda46f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:644f31b8f04ac36efae482c4dba0e159d90f726e7137d0844daed693b094fbb2 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab3dd4468f61f048828cdaa586e03c5abb891452 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74c3b823b6d9e255092c6da34db98dfde2095f3901ab03e6e9780692a905196b +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c253ac539efe1f21ae9d9d952819f65e2388d31 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b19eb0d7eb62186650603736dd968ffdeab1920c67ff66ae2968a936842468f +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ac4d4f69922a5e1e8db01e72921afca765277d2 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:115d383e8e6428d18252146a596c49af13fb1c331648fea0e0b9e902f3c2dd72 +size 199058647 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b72f5205bd399d80e20455806cd385f810be1fb1 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e9185d660a58eaca956c09b230e59d0a84fa2544f0eac4bba498217f4c07c7f +size 199058647 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..964373e52f97b1ff2454afc45172721ce317ffba --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd4f8a6056fd4b2406da16a7ad9d051cd37a5189a3d6b68f68e6c21fdea90620 +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..190b1464a6e7d4be497b55b7c3e16275d5735e75 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c6aaf11945f515614eab76a43e93d3b6aee9ee33c68752803fa98b7eccd1db0 +size 199058850 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00afe1143b2d5b5d56d98cb614cde7e6f7b84bf5 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4c7f69bcd121dee43236a9de248d611feab6682d24c0aa440d9e3db8b63553f +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f3a6e923a16ccad8963fa11c67394fa3867cf33 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14d858c85d4c433277c5f40222be37fd81643f5085b6f7a6da61856722663f73 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..412c22ee9c63a66042c27dcf9e5c28f3b47923cf --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65a170b32bdc1b2e9c88ba55e3d1154b401d596781aef3bee49717083f658e90 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc65faed810339898bb56b85382d8d4e3bd9ac3f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a304ca07bc26a9d3f14c95fe8b4428dc955645f7669cc738f1dcaac294ee27ce +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3f54dafd590c27840938f0d21fd454c91b71819 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d38a916fa284914b767ff55981cf9603bae8376c121b8c6c8eb1ad09cc2d0f1 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dfa970ef7b7995415dc3055cb1db615afed382d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b07de0390c497862ff8d157a05b39b3d3f5e906e163ca4a12a8e42c7017ec0fe +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce0de80381d596759fb9b1e225990f0acca243d1 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3d5ea049169f78d223129bb0784f061882e0ab075fbc1e14d22c419c6775a9a +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad020750269309bcaa482453723d69e6059ab96d --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c5afad97c23e83f474df18f42194333d439a71a0b5f75033282c4db6c48ca7c +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92bca2fed6441f7c91928dd5893164f208e57793 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00abaded1b135c8a90733cd78a6f518451c42b35e351692859cd0e89a0ba5648 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f56262ba511ad0610d6fc1f33fbf0995ab4d3c4 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:137c42a680f5108ae929faae72f6d7d3ef63b49633d2c528e4ced46beff90ee2 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5cd39f16d119be46723c286f7442f5276c09d97f --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a42b01fbf2e9179287c53568e88bc3400ce24393d6fe5e1a898277695ee83379 +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..097ecf179223ec5b6cc392c32b642082bec313c2 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c2cd5ce9093ab033e3d99ef47e0ca0173e83eee8e8ff939765a003fd156867f +size 199058722 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..34ab27b49588254905588f0dacb45fec2d29e149 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:958c997c6cbca12a28a8fc8d1d8c3eeecd7ee22415f5c2a9e7292cb2cf4f05f1 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..53507092f086b3656da393dcaf5ae16d48ddc4db --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07cd9bd6147842af6df8411aa89020810283e8464f627e8c04067805b6f14dc9 +size 199058786 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03edf527a935b2c62829c237c04089292bebabc2 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d8b6b11323a7e920ddbc81ef3ad24faeb681f0a2b007664895574c34c4e78cb +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b21fc75cdb214276ca225fe8fde1a7c95a0af274 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bf58233746843ed6157527edf924faee17736547e984dc4dfa75613243607c7 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..82c49b09177ff25cfd1d9d20913b164d88084551 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:128e7ae3606b318e6bd10b73c1be1d1f139895082083a08c62998d1079c0c98d +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c67744031678253a2e5288944d92e6208a67cb03 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cce89ab46e6cc33c2811ff5d18c1704e2357600b186edaed123d2a1227d8e40 +size 199058658 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4733c651cd254864ab3a66420eccbbd5aeb858c1 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a20f8ac8ce8a5e66ffe7b6526e911085ab81f657a82551f8e83a3ed4f7bebf44 +size 199058839 diff --git a/perplexity25/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/perplexity25/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b64fbbaed90ea585e9aa33a8d07fa5ab63ca4862 --- /dev/null +++ b/perplexity25/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ce1bbabc6f18ba73f34009a3218de868a389acbf6bde6630ed0a7c7106b18dd +size 199058839 diff --git a/perplexity25/global_step80108/layer_01-model_00-model_states.pt b/perplexity25/global_step80108/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50d3b56099291b4c2b72a3fc34f1b6b2dc03528e --- /dev/null +++ b/perplexity25/global_step80108/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c223d027597688c410d2c39fd6a6aaf1c6ed187442a318b9f2e1263501e9869 +size 167511299 diff --git a/perplexity25/global_step80108/layer_01-model_01-model_states.pt b/perplexity25/global_step80108/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30f191aefb1446f313d832086eb10d1b6bbc7a15 --- /dev/null +++ b/perplexity25/global_step80108/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49b9e2d7fcee26508e264c8ddc604eb4362ee283bc0ea7d31701c692c90aa67d +size 167511299 diff --git a/perplexity25/global_step80108/layer_03-model_00-model_states.pt b/perplexity25/global_step80108/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..15f136fc1a63bd5755b2f694d5366b8f85712292 --- /dev/null +++ b/perplexity25/global_step80108/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eb46975a23bcb654a46bcc15be4b21746a99e19de2fd63fcca358d2197ec8b9 +size 113308931 diff --git a/perplexity25/global_step80108/layer_03-model_01-model_states.pt b/perplexity25/global_step80108/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e662142d58497c8f40110d17aee9d153308fb5e --- /dev/null +++ b/perplexity25/global_step80108/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f79b7dce53e191353d07dd2e98ba17eae7913282367d931ddad152890596276a +size 113308931 diff --git a/perplexity25/global_step80108/layer_04-model_00-model_states.pt b/perplexity25/global_step80108/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0084c090dd32c8f0f35f26b0de2e4791d4f28865 --- /dev/null +++ b/perplexity25/global_step80108/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c5f9f04091db4aa39cc8db44cacf57e222671700983eb66846a47f0cdaf5c2c +size 113308931 diff --git a/perplexity25/global_step80108/layer_04-model_01-model_states.pt b/perplexity25/global_step80108/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37e4e2cfe2d5d460738f242ce4d7dd140bb49aa2 --- /dev/null +++ b/perplexity25/global_step80108/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dca01964ed8714a26b038f47d08f2d6a38662e9309ea8632c9edc3ed30b59d2d +size 113308931 diff --git a/perplexity25/global_step80108/layer_05-model_00-model_states.pt b/perplexity25/global_step80108/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d9697475403da3ea711f60e282cba2eaf975d5b --- /dev/null +++ b/perplexity25/global_step80108/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45d3e3d040dbb264ff069391aa29bd92cc90f2df7bdd501a244a1cee372f349f +size 113308931 diff --git a/perplexity25/global_step80108/layer_05-model_01-model_states.pt b/perplexity25/global_step80108/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1f4c964930d44ce315db5cee59b46d84b4408e2 --- /dev/null +++ b/perplexity25/global_step80108/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12e322cc6bd57f15fb2c4b91247ed5c577ca05163c244e4a7d1d05541f73a426 +size 113308931 diff --git a/perplexity25/global_step80108/layer_06-model_00-model_states.pt b/perplexity25/global_step80108/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de33b0ca4a47d4de318f4dd69af3f61a86e89175 --- /dev/null +++ b/perplexity25/global_step80108/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaf0a742519897d7d0c847e81f695bc8122db28b670ec6b09e627b06acabb43e +size 113308931 diff --git a/perplexity25/global_step80108/layer_06-model_01-model_states.pt b/perplexity25/global_step80108/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fed433cb54e1821552a51d40154c6c4e76ddccc --- /dev/null +++ b/perplexity25/global_step80108/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:727a9bb5587c3cc4aabe2283b18b075107009e0a8805714b8c5bca1c019e461c +size 113308931 diff --git a/perplexity25/global_step80108/layer_07-model_00-model_states.pt b/perplexity25/global_step80108/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61e43bb2e22232475b47b10e58ada4f1136821cd --- /dev/null +++ b/perplexity25/global_step80108/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fab7e4044ecb3495f262e9247075965e2aac46901f7e09636e9cc0b280d57765 +size 113308931 diff --git a/perplexity25/global_step80108/layer_07-model_01-model_states.pt b/perplexity25/global_step80108/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21c5d434c35679bf2e4cd66126848a36267127c7 --- /dev/null +++ b/perplexity25/global_step80108/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:823936e7e53a9a59addfee62a0ba2d824ed4c39e7b8df8b3ef9c4a70e378e904 +size 113308931 diff --git a/perplexity25/global_step80108/layer_08-model_00-model_states.pt b/perplexity25/global_step80108/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dd5272e6a51cb6948be5ea17b7061c394faadd0 --- /dev/null +++ b/perplexity25/global_step80108/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:064caf18f62f3887815c9d1ed5b245796732bda53575ecfc54a6e749ee3d5ea5 +size 113308931 diff --git a/perplexity25/global_step80108/layer_08-model_01-model_states.pt b/perplexity25/global_step80108/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12e28a30a377e7c7f38fb661673972c7fb72d718 --- /dev/null +++ b/perplexity25/global_step80108/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e51af4bbf15de0defedb97051465fc4d24f67ed3d69af5744f6cc7428432f08 +size 113308931 diff --git a/perplexity25/global_step80108/layer_09-model_00-model_states.pt b/perplexity25/global_step80108/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..78f5b245fd43fb0e792669476fe010d632e026f5 --- /dev/null +++ b/perplexity25/global_step80108/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5fe64bce086aecbfbeb3e5f679294f25dda95e922e0317ff04c721dce2c3a45 +size 113308931 diff --git a/perplexity25/global_step80108/layer_09-model_01-model_states.pt b/perplexity25/global_step80108/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86027e9568de2e12fbf9e25d3fdb5d531a811f79 --- /dev/null +++ b/perplexity25/global_step80108/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16d733908f10f64b82770e3672086777164481212d1b9bc5d10b14d13016745b +size 113308931 diff --git a/perplexity25/global_step80108/layer_10-model_00-model_states.pt b/perplexity25/global_step80108/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b89768db8a2e55832c9db82d41b21e0decaffd25 --- /dev/null +++ b/perplexity25/global_step80108/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e155b0ab9f99c7a3894679e929f7859b03cd4291f6cfbead5441f2e94220dff +size 113308931 diff --git a/perplexity25/global_step80108/layer_10-model_01-model_states.pt b/perplexity25/global_step80108/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..26a03d6de61c882c0c7ec00f33f589a00b1fb038 --- /dev/null +++ b/perplexity25/global_step80108/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b4605d519f83aaf1b746e5d97a81192f1e95516c3a53735f31e37944743f22e +size 113308931 diff --git a/perplexity25/global_step80108/layer_11-model_00-model_states.pt b/perplexity25/global_step80108/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d228cfe9ad765ee53ee56c3fdfd2d79515c8da9d --- /dev/null +++ b/perplexity25/global_step80108/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7376444b98a46410bf23eb7fc727185da92d0ac41c27fe50d2a61084545ea61c +size 113308931 diff --git a/perplexity25/global_step80108/layer_11-model_01-model_states.pt b/perplexity25/global_step80108/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c4587727aaa027c5d38765778b3213a827f1998 --- /dev/null +++ b/perplexity25/global_step80108/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1b928109bf0ff231c4327ac804795137cffcc03c7adeb7bab2d1fdda1c4a573 +size 113308931 diff --git a/perplexity25/global_step80108/layer_12-model_00-model_states.pt b/perplexity25/global_step80108/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..976c80b9d3af6fbb204d6b45477356357eed8738 --- /dev/null +++ b/perplexity25/global_step80108/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c453652c90cc68d0620cd0a5780f18214a6cb63886bd95366cb0710f18c37dc +size 113308931 diff --git a/perplexity25/global_step80108/layer_12-model_01-model_states.pt b/perplexity25/global_step80108/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe43ce08894641f55940a92d0bb89fcfafabd23f --- /dev/null +++ b/perplexity25/global_step80108/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:084d4ecaa73a5bd5675da193e52f27a0d7c9fb4f6f815bfd95ccba78a89207c8 +size 113308931 diff --git a/perplexity25/global_step80108/layer_13-model_00-model_states.pt b/perplexity25/global_step80108/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3557fc0293ebde4c1734b64b3f62012ec55ccfe --- /dev/null +++ b/perplexity25/global_step80108/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76ccb9575c72f524f7a3085742b25b76629fa824494a85975c0ebb1c036a558e +size 113308931 diff --git a/perplexity25/global_step80108/layer_13-model_01-model_states.pt b/perplexity25/global_step80108/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..090d676a8eb416b9eec84006f955442760b80d11 --- /dev/null +++ b/perplexity25/global_step80108/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2570a71c725ff3fc3b599f5787eb218443a844c930ea3be52fc4d4cd3d466cfb +size 113308931 diff --git a/perplexity25/global_step80108/layer_14-model_00-model_states.pt b/perplexity25/global_step80108/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a0a4e8614a1131dd3c74573ed98bb8be085ab4b --- /dev/null +++ b/perplexity25/global_step80108/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e03d172edb57912283a3ce4121bc998341c41650942acd8bdc641ee5eeca9be +size 113308931 diff --git a/perplexity25/global_step80108/layer_14-model_01-model_states.pt b/perplexity25/global_step80108/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43e19a85485854e720d6b01467821d92a8d62f6f --- /dev/null +++ b/perplexity25/global_step80108/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb351e8e723bfcbf27a10d4e3698ed60baaa4b45181d3a241832c7bed8045906 +size 113308931 diff --git a/perplexity25/global_step80108/layer_15-model_00-model_states.pt b/perplexity25/global_step80108/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d517da67054ef3f0f77309a8f24a83a6586b3bc8 --- /dev/null +++ b/perplexity25/global_step80108/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5477b2d1b48fa0f84bda9b45c715e75a05123982e5db5b1ce6afe53ee6d7199e +size 113308931 diff --git a/perplexity25/global_step80108/layer_15-model_01-model_states.pt b/perplexity25/global_step80108/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..582c426206769f7c215fefbfe2d43d9b838b5ead --- /dev/null +++ b/perplexity25/global_step80108/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b384735764e3c66789ab809b816534a69822c6e60e034526d4f40053d429f1e3 +size 113308931 diff --git a/perplexity25/global_step80108/layer_16-model_00-model_states.pt b/perplexity25/global_step80108/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9abd9936a1d950ef6e32c52b36a608c4eeb61d43 --- /dev/null +++ b/perplexity25/global_step80108/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e759b2fa94c36b1d48344b1c7fae27404e4df9856f34ecd98e33da5f732b7e2 +size 113308931 diff --git a/perplexity25/global_step80108/layer_16-model_01-model_states.pt b/perplexity25/global_step80108/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f06e8ea143b09b99bd871d733e26f88654f6cc1 --- /dev/null +++ b/perplexity25/global_step80108/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fff48711268d031a6727610cfc54d6a8a19fdf6dfb0a5f758e095715812def5 +size 113308931 diff --git a/perplexity25/global_step80108/layer_17-model_00-model_states.pt b/perplexity25/global_step80108/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ef96ef0d621c5ab35318b82e7d980cbe5cabbe4 --- /dev/null +++ b/perplexity25/global_step80108/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b7c1273657ffa4812aa55a0b0cc5c7ce3c778a23f73668e46906aca21e0b7fc +size 113308931 diff --git a/perplexity25/global_step80108/layer_17-model_01-model_states.pt b/perplexity25/global_step80108/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a302598bbe69ef2b7b6f9c46ccd01c9b874c7d3c --- /dev/null +++ b/perplexity25/global_step80108/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:596b1df217e858f6af10aa567206090247b09189e67257e8554d92081e8bff4f +size 113308931 diff --git a/perplexity25/global_step80108/layer_18-model_00-model_states.pt b/perplexity25/global_step80108/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c65bdaf570b37b1db39262c74677d6adc2af57da --- /dev/null +++ b/perplexity25/global_step80108/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b0f51a13ad8e482c85446790080859d7f4a4d05ff133fdc95f5f185be4e4970 +size 113308931 diff --git a/perplexity25/global_step80108/layer_18-model_01-model_states.pt b/perplexity25/global_step80108/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd6cba1818c7cba3c514ae3579b21412112eebe8 --- /dev/null +++ b/perplexity25/global_step80108/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0672bbdd5a9b857fc88824b3056d084dbc7c8ec69471580b7c37277acef4c27 +size 113308931 diff --git a/perplexity25/global_step80108/layer_19-model_00-model_states.pt b/perplexity25/global_step80108/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92f8ee8c90c4355c667b40f8e7a507216e55c7b1 --- /dev/null +++ b/perplexity25/global_step80108/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c01e693c8c353d19cc7590e17d64b44160197b7df4907489aa137dd16e0d19b +size 113308931 diff --git a/perplexity25/global_step80108/layer_19-model_01-model_states.pt b/perplexity25/global_step80108/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c63d5ecec9b4f3dd542cf5435c626cf6655d20f --- /dev/null +++ b/perplexity25/global_step80108/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfc31fcf0be8c4d6c0daedbd1d4bb1b0caf19a0262a27b13c084af356f464919 +size 113308931 diff --git a/perplexity25/global_step80108/layer_20-model_00-model_states.pt b/perplexity25/global_step80108/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..559079e63151fe82d6979ca0d01704b5dc878fce --- /dev/null +++ b/perplexity25/global_step80108/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0316bff6836940b1513c4eafcb8a758cbc154848429c061e964c2b7f3b6fcc53 +size 113308931 diff --git a/perplexity25/global_step80108/layer_20-model_01-model_states.pt b/perplexity25/global_step80108/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05245f2a9bd274a191a61e8ab0c4a42c777a8503 --- /dev/null +++ b/perplexity25/global_step80108/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb0d47d097ee8b6f09cedab4bfbab2a69f30c08dfe56444a2f788534740638ad +size 113308931 diff --git a/perplexity25/global_step80108/layer_21-model_00-model_states.pt b/perplexity25/global_step80108/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..893851a0788ac72ffe585f64c798c0386d25e8e7 --- /dev/null +++ b/perplexity25/global_step80108/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0568300adad1286b1b2460d885a3be0af01edecf7bf59b55358413220f32b7d8 +size 113308931 diff --git a/perplexity25/global_step80108/layer_21-model_01-model_states.pt b/perplexity25/global_step80108/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b432b87b75d41d7cfd69186c9662d5828e57b008 --- /dev/null +++ b/perplexity25/global_step80108/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90be5f2979b2a108e1db71ce0b9030c7595ceab57a1b0d212424ac82f2fc90c4 +size 113308931 diff --git a/perplexity25/global_step80108/layer_22-model_00-model_states.pt b/perplexity25/global_step80108/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff141b0b494e5549f693954ecb04879729f1e500 --- /dev/null +++ b/perplexity25/global_step80108/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbd1f6e28dc02d792f87a2e95b47dba8954a4e0cec33a5d78289c1c261b3ac9d +size 113308931 diff --git a/perplexity25/global_step80108/layer_22-model_01-model_states.pt b/perplexity25/global_step80108/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5fcf6933aadcde0076d3672700aa236a29276237 --- /dev/null +++ b/perplexity25/global_step80108/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb6be8ae7b937f2c7ae6a71766d86adfd8440cfe8d708e45a40d4fc523bd0342 +size 113308931 diff --git a/perplexity25/global_step80108/layer_23-model_00-model_states.pt b/perplexity25/global_step80108/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb1a68e64416cf25d8e2655c5f549e0725c95216 --- /dev/null +++ b/perplexity25/global_step80108/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc6130fb9240be6cbda0ace7e2b27b8104037e26dda6ac0705000304ffe9e0dc +size 113308931 diff --git a/perplexity25/global_step80108/layer_23-model_01-model_states.pt b/perplexity25/global_step80108/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c65542311ab9fc826d5a0d5e9f59a9612a0f83ce --- /dev/null +++ b/perplexity25/global_step80108/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:369919b22f45f64b8a7f3c7477e2abf14d9f0397b79a3722ad7e2da7cf4ed618 +size 113308931 diff --git a/perplexity25/global_step80108/layer_24-model_00-model_states.pt b/perplexity25/global_step80108/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62da1031c8a0a399ff8a04d7f53e9a9cf6d0f6eb --- /dev/null +++ b/perplexity25/global_step80108/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da8df587a3566f6501b07ee1f9c999402c7d684b2fb8de49edbee558109ca4c6 +size 113308931 diff --git a/perplexity25/global_step80108/layer_24-model_01-model_states.pt b/perplexity25/global_step80108/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c3db522a2d61a7e3531286f5e995063b298379a --- /dev/null +++ b/perplexity25/global_step80108/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a55ebf0bab87e18a8e86ef2c396a798acab4378988f128eb80abda0730343982 +size 113308931 diff --git a/perplexity25/global_step80108/layer_25-model_00-model_states.pt b/perplexity25/global_step80108/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a27fbce95e5e5b2c2b67464765414ebc5456044 --- /dev/null +++ b/perplexity25/global_step80108/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75751608db387813d1fe0fa733e26729ca3ad8728943f61c3cd3aa0aa105ddc1 +size 113308931 diff --git a/perplexity25/global_step80108/layer_25-model_01-model_states.pt b/perplexity25/global_step80108/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99a12ac76be37bd13e9938b3072e29a8acb3f434 --- /dev/null +++ b/perplexity25/global_step80108/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cddaf98ee1914d7eac98c31a165eebd49f292b82ae5289dbd0bfd77ab764073f +size 113308931 diff --git a/perplexity25/global_step80108/layer_26-model_00-model_states.pt b/perplexity25/global_step80108/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9cd0ff2c326e3d8748703283933af759f681dbc --- /dev/null +++ b/perplexity25/global_step80108/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b2ab3d7b89755b20ee2a3cae80c61cec7a556f8eca7057e420f3bfeab22bd86 +size 113308931 diff --git a/perplexity25/global_step80108/layer_26-model_01-model_states.pt b/perplexity25/global_step80108/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe442473d489df3980db1a2287b954463de26ceb --- /dev/null +++ b/perplexity25/global_step80108/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49d2675404235941dba3b671000f685bdbddd2f9f5ce7def6da8f07ea01b89d1 +size 113308931 diff --git a/perplexity25/global_step80108/layer_27-model_00-model_states.pt b/perplexity25/global_step80108/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d30c4cb361d5dbe56467c218805d975204ae567 --- /dev/null +++ b/perplexity25/global_step80108/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbe4081d8d310ab9ffa546da1b9e62d75b004a85d541f6c05c6b04878cf1e6fc +size 113308931 diff --git a/perplexity25/global_step80108/layer_27-model_01-model_states.pt b/perplexity25/global_step80108/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f3ff4af881053be208231e24bde14de5e6dd376 --- /dev/null +++ b/perplexity25/global_step80108/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dbb95605df59f27a80a200298f4b8646bede03eb99b7b3fe950779693afc597 +size 113308931 diff --git a/perplexity25/global_step80108/layer_28-model_00-model_states.pt b/perplexity25/global_step80108/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32a0e3d4a1956c013a7f35cc78eb64333342a1ce --- /dev/null +++ b/perplexity25/global_step80108/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712578453a6ac3c43dfc01e1b64cf97f0a20ddedc41ab50c4eb09dc90e59bed7 +size 113308931 diff --git a/perplexity25/global_step80108/layer_28-model_01-model_states.pt b/perplexity25/global_step80108/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..385e100073a5841cd110a66bd2b58b6b3dec5c6d --- /dev/null +++ b/perplexity25/global_step80108/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:834f38529bf8320020b067058699af9efcb2b907df0c4affac523a184ebef3c9 +size 113308931 diff --git a/perplexity25/global_step80108/layer_29-model_00-model_states.pt b/perplexity25/global_step80108/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c669f5fd0d38df8a63f4dd490fb05e0d35dc5c4 --- /dev/null +++ b/perplexity25/global_step80108/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d1198bc7ab27b8c4b1c8a7257fbb792bcb60b1196081ee5dc906a135ccd073d +size 113308931 diff --git a/perplexity25/global_step80108/layer_29-model_01-model_states.pt b/perplexity25/global_step80108/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9dcfbd84aec4d6aee0e3f138242389056e65f9c --- /dev/null +++ b/perplexity25/global_step80108/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f6be52a7cad455bd3b57d206d78439bcd61af72423768d4213dd3f53ef27814 +size 113308931 diff --git a/perplexity25/global_step80108/layer_30-model_00-model_states.pt b/perplexity25/global_step80108/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8430027e64bf90b89cdaed9bad06dc3c1d8d582e --- /dev/null +++ b/perplexity25/global_step80108/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6110bffb5c3073d0b4fe89bd01cafd60de45f13d3a2b7fa0bb57cf294e4a4152 +size 113308931 diff --git a/perplexity25/global_step80108/layer_30-model_01-model_states.pt b/perplexity25/global_step80108/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..981b7c507cfa2f856cbd2883debd9bc026fdd526 --- /dev/null +++ b/perplexity25/global_step80108/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34595adb9883f156f7d4699b396bbf2049fa092edca805f634637d70337a60dd +size 113308931 diff --git a/perplexity25/global_step80108/layer_31-model_00-model_states.pt b/perplexity25/global_step80108/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d93582b8638854571c1ec0f90117cce5b625a249 --- /dev/null +++ b/perplexity25/global_step80108/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:771ac999aa0d68eb7ab1dbc70680314c3c5a8794a8cd9b05792db65d74098406 +size 113308931 diff --git a/perplexity25/global_step80108/layer_31-model_01-model_states.pt b/perplexity25/global_step80108/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6a9841abd5c779e3f582d60a5bea4d52dd2aada --- /dev/null +++ b/perplexity25/global_step80108/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10b1baf5a1350fc8b515b89a57d2ef2ce3cd58bd6e8d46201fc44aca2a476052 +size 113308931 diff --git a/perplexity25/global_step80108/layer_32-model_00-model_states.pt b/perplexity25/global_step80108/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06225784f98c9ba9a9b09379b95dea6da6c6d150 --- /dev/null +++ b/perplexity25/global_step80108/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a2f75dda78e71c267297edc4bb5ff6b94029cf0ffe97c429bb0b1466655cd8b +size 113308931 diff --git a/perplexity25/global_step80108/layer_32-model_01-model_states.pt b/perplexity25/global_step80108/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..33a6fe652480b194f67a1fa6b96950e5a151d0fc --- /dev/null +++ b/perplexity25/global_step80108/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e8f05519f30871f41bb4a27923a29581433ec68faab4a7fd6a7cacc24c9bc73 +size 113308931 diff --git a/perplexity25/global_step80108/layer_33-model_00-model_states.pt b/perplexity25/global_step80108/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..75ca55255687c503ec79de7991179e2e60472c73 --- /dev/null +++ b/perplexity25/global_step80108/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:981abcd87840b8be57fc8ba9e6913527bf0b58b6e525ba5c461c90c3fd493b91 +size 113308931 diff --git a/perplexity25/global_step80108/layer_33-model_01-model_states.pt b/perplexity25/global_step80108/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cec121b05fa8622dc2b28e861589a5d219392f74 --- /dev/null +++ b/perplexity25/global_step80108/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dca9fee440812ef7691cfb61dca836ebb401efc1cb16da64dc7f6f13175e1fbf +size 113308931 diff --git a/perplexity25/global_step80108/layer_34-model_00-model_states.pt b/perplexity25/global_step80108/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9406afb8248c1113a486d50432070cac0f3701b9 --- /dev/null +++ b/perplexity25/global_step80108/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f159d57f9e2ba50684d481ca3541a6b3932abb72f0b0e7f12300fd2858fa5f01 +size 113308931 diff --git a/perplexity25/global_step80108/layer_34-model_01-model_states.pt b/perplexity25/global_step80108/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9dc9365112a07802e35777a76f9c4a8cdda69113 --- /dev/null +++ b/perplexity25/global_step80108/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edbd2f6a97f9fe71ec27c7d8b31abce01fd9314bf7dbe7aa0c6a8e08f35533eb +size 113308931 diff --git a/perplexity25/global_step80108/layer_35-model_00-model_states.pt b/perplexity25/global_step80108/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8eefdb36d688458cae19bee3d01b974cd7e5855d --- /dev/null +++ b/perplexity25/global_step80108/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27a015b6b747d2f1edacf09c1754d864bc70bd421a58a4b2d43883da7edc543c +size 113308931 diff --git a/perplexity25/global_step80108/layer_35-model_01-model_states.pt b/perplexity25/global_step80108/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..327ad6ac4a93ce5153a4fc0886dd81fe7a010a48 --- /dev/null +++ b/perplexity25/global_step80108/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f722855f5ffdb60a7f9d2a23e37ed06f81b3146f5d0533a1d0d61979f753111 +size 113308931 diff --git a/perplexity25/global_step80108/layer_36-model_00-model_states.pt b/perplexity25/global_step80108/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03d8b67c87cb6ed1bd69de338cbbb1c9f167a1e8 --- /dev/null +++ b/perplexity25/global_step80108/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea5887cfb118469c8ac4694a5e711765ccd9357ea393ea7c770e0d5d8b068103 +size 113308931 diff --git a/perplexity25/global_step80108/layer_36-model_01-model_states.pt b/perplexity25/global_step80108/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5b11c2d1cf24a045e95d470c6efc202f7a6a81f --- /dev/null +++ b/perplexity25/global_step80108/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d13b3a584ba1492b945f4d1d6feff5c2cca4308423c03195b043f0320309e8b3 +size 113308931 diff --git a/perplexity25/global_step80108/layer_37-model_00-model_states.pt b/perplexity25/global_step80108/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0c1dd673bdc599419b4ae21255f9f78d2cfdff0 --- /dev/null +++ b/perplexity25/global_step80108/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a275382523fa5b9a55ff1ec2d88f1438c20b0cd9eeb21c105058eecd3f420d1 +size 113308931 diff --git a/perplexity25/global_step80108/layer_37-model_01-model_states.pt b/perplexity25/global_step80108/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c48e6d865627c109a3815075d791d9c920d9e905 --- /dev/null +++ b/perplexity25/global_step80108/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ae5cea3f66a3d6d4ab22973e5d7a39086c06e4e775790b8520c780f738476f1 +size 113308931 diff --git a/perplexity25/global_step80108/layer_38-model_00-model_states.pt b/perplexity25/global_step80108/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f486690cb5e7a1df651e87ff3b22c44fe5403dc7 --- /dev/null +++ b/perplexity25/global_step80108/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:589038d8388058aee7baf3c61790466a9ee7db5d3e9aa4df19168f64784e6e35 +size 113308931 diff --git a/perplexity25/global_step80108/layer_38-model_01-model_states.pt b/perplexity25/global_step80108/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..49da64050a9151e202ffe91ed5d39fe517ee3731 --- /dev/null +++ b/perplexity25/global_step80108/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3136fae17662c6984205e36e2ac292a253fd952663303804e1d354746da9bd16 +size 113308931 diff --git a/perplexity25/global_step80108/layer_40-model_00-model_states.pt b/perplexity25/global_step80108/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03e3b274396ca07d2a467d165cba6e68f376044d --- /dev/null +++ b/perplexity25/global_step80108/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ebd82e21789eb620f60fa881f83b323128235cb9dfa1c6f9ef1f05303798159 +size 13507 diff --git a/perplexity25/global_step80108/layer_40-model_01-model_states.pt b/perplexity25/global_step80108/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..603d87e9cc6bf0a26c9142aff798f7f96acd399f --- /dev/null +++ b/perplexity25/global_step80108/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26ea0ae13a368879d0cb773e311751eb8062d282ac2cf07712cbafe661575a54 +size 13507 diff --git a/perplexity25/global_step80108/mp_rank_00_model_states.pt b/perplexity25/global_step80108/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c1dc95f67c04dc88f6534e8ff9db6600c29144c --- /dev/null +++ b/perplexity25/global_step80108/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eb51b9e4c8d9fe73ed5037dcd60b6879eaeb76edfcb7955e37122b67ba5f5f7 +size 51507 diff --git a/perplexity25/global_step80108/mp_rank_01_model_states.pt b/perplexity25/global_step80108/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c0d0bfe094ec3c5ed6da1bbf2fea7e7ae1589e8 --- /dev/null +++ b/perplexity25/global_step80108/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53c653d25d10c56c799619a12e9f99aa2c321267d69285a31d485661c5d028ee +size 51507 diff --git a/perplexity25/sbatch_4b284bc4perplexity25.sh b/perplexity25/sbatch_4b284bc4perplexity25.sh new file mode 100644 index 0000000000000000000000000000000000000000..4c2386dfd10fea00286aa4136672390dcc3cd315 --- /dev/null +++ b/perplexity25/sbatch_4b284bc4perplexity25.sh @@ -0,0 +1,163 @@ +#!/bin/bash +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=40 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + + +VARIANT=4b284bc4perplexity25 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +DATA_PATH="/scratch/project_462000119/data/c4perplexity/gpt2tok_perplexity_text_document" +TRAIN_DATA_PATH=trainperplexity25.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4perplexity/gpt2tok_c4_en_pplx_25_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=2 + +MICRO_BATCH_SIZE=2 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_4516M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=10000 + +# Tokens: 84_000_000_000 +# -> Samples: 41_015_625.0 +TRAIN_SAMPLES=41_015_625 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.95 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 410_156 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/perplexity25/sbatch_4b284bc4perplexity25val.sh b/perplexity25/sbatch_4b284bc4perplexity25val.sh new file mode 100644 index 0000000000000000000000000000000000000000..d2df6b001aae5be3aa26a877d2ea21662b199f41 --- /dev/null +++ b/perplexity25/sbatch_4b284bc4perplexity25val.sh @@ -0,0 +1,168 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=40 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=4b284bc4perplexity25val +VARIANT_CKPT=checkpoints_4b284bc4perplexity25 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=$VARIANT_CKPT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train1b5.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=2 + +MICRO_BATCH_SIZE=2 +GRADIENT_ACCUMULATION_STEPS=2 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_4516M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 25611230000 +# -> Samples: 12505484 +TRAIN_SAMPLES=1 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 0 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + --override-lr-scheduler \ + --reset-progress \ + --no-load-optim \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1 \ + --eval-iters 100 \ + --eval-only true \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun_32.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683712349.nid006831.112858.0 b/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683712349.nid006831.112858.0 new file mode 100644 index 0000000000000000000000000000000000000000..eb89da41601748a07eb68702644baa2c53225b17 --- /dev/null +++ b/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683712349.nid006831.112858.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5f906c7030cc77efd8b04274c095fab2c8667d6c6b9b78f3655e217494d5e7d +size 51833827 diff --git a/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683793006.nid006098.98910.0 b/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683793006.nid006098.98910.0 new file mode 100644 index 0000000000000000000000000000000000000000..739d2c7f41b050e1fffc3279fc9ab8187933db18 --- /dev/null +++ b/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683793006.nid006098.98910.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a38f52ade244ffbea5279f0467f87c8e39fb44b2388ef9be8bc50e9bf84a0cc4 +size 107974906 diff --git a/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683958717.nid006098.33981.0 b/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683958717.nid006098.33981.0 new file mode 100644 index 0000000000000000000000000000000000000000..271ae68c386cc77839d305adb4252ccce062f2e4 --- /dev/null +++ b/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683958717.nid006098.33981.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51ac8133453345e304a4082ec98e6c542ac46b2d35647ea07b1f180a3a589027 +size 16424 diff --git a/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683959028.nid006037.96130.0 b/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683959028.nid006037.96130.0 new file mode 100644 index 0000000000000000000000000000000000000000..28150da7475c52ca95eb26a1dd94744c33616c5b --- /dev/null +++ b/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683959028.nid006037.96130.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fcef7bfab8cbb19188832a41151fd97b5b2660f56b2b6af5174001f4a0ee126 +size 16424 diff --git a/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683959343.nid006098.43443.0 b/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683959343.nid006098.43443.0 new file mode 100644 index 0000000000000000000000000000000000000000..bb5a49b43b745497b35f7cba94fa573501bfaae3 --- /dev/null +++ b/perplexity25/tensorboard_4b284bc4perplexity25/events.out.tfevents.1683959343.nid006098.43443.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:456c0a74e8a0ff64ebd0f2c5522ecee5fc614554440326bc201c7cd5505d237a +size 16424 diff --git a/perplexity25/tensorboard_4b284bc4perplexity25val/events.out.tfevents.1683959851.nid006989.19807.0 b/perplexity25/tensorboard_4b284bc4perplexity25val/events.out.tfevents.1683959851.nid006989.19807.0 new file mode 100644 index 0000000000000000000000000000000000000000..bc16b3990e6d8a0a12d10e293cfa6642da3b34b5 --- /dev/null +++ b/perplexity25/tensorboard_4b284bc4perplexity25val/events.out.tfevents.1683959851.nid006989.19807.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5988e99574af63af80c199beae69c39d3729e4bcf9da6fb138217e3a9201bd3 +size 980 diff --git a/perplexity25/transformers/config.json b/perplexity25/transformers/config.json new file mode 100644 index 0000000000000000000000000000000000000000..df814c3a71ee49044410636b5eeb4f43611690ba --- /dev/null +++ b/perplexity25/transformers/config.json @@ -0,0 +1 @@ +{"vocab_size": 50432, "n_positions": 2048, "n_embd": 3072, "n_layer": 36, "n_head": 24, "n_inner": 12288, "activation_function": "gelu", "resid_pdrop": 0.1, "embd_pdrop": 0.1, "attn_pdrop": 0.1, "layer_norm_epsilon": 1e-05, "initializer_range": 0.02, "summary_type": "cls_index", "summary_use_proj": true, "summary_activation": null, "summary_first_dropout": 0.1, "summary_proj_to_labels": true, "scale_attn_weights": true, "use_cache": true, "scale_attn_by_inverse_layer_idx": false, "reorder_and_upcast_attn": false, "bos_token_id": 50256, "eos_token_id": 50256, "return_dict": true, "output_hidden_states": false, "output_attentions": false, "torchscript": false, "torch_dtype": null, "use_bfloat16": false, "tf_legacy_loss": false, "pruned_heads": {}, "tie_word_embeddings": true, "is_encoder_decoder": false, "is_decoder": false, "cross_attention_hidden_size": null, "add_cross_attention": false, "tie_encoder_decoder": false, "max_length": 20, "min_length": 0, "do_sample": false, "early_stopping": false, "num_beams": 1, "num_beam_groups": 1, "diversity_penalty": 0.0, "temperature": 1.0, "top_k": 50, "top_p": 1.0, "typical_p": 1.0, "repetition_penalty": 1.0, "length_penalty": 1.0, "no_repeat_ngram_size": 0, "encoder_no_repeat_ngram_size": 0, "bad_words_ids": null, "num_return_sequences": 1, "chunk_size_feed_forward": 0, "output_scores": false, "return_dict_in_generate": false, "forced_bos_token_id": null, "forced_eos_token_id": null, "remove_invalid_values": false, "exponential_decay_length_penalty": null, "suppress_tokens": null, "begin_suppress_tokens": null, "architectures": ["GPT2LMHeadModel"], "finetuning_task": null, "id2label": {"0": "LABEL_0", "1": "LABEL_1"}, "label2id": {"LABEL_0": 0, "LABEL_1": 1}, "tokenizer_class": null, "prefix": null, "pad_token_id": null, "sep_token_id": null, "decoder_start_token_id": null, "task_specific_params": null, "problem_type": null, "_name_or_path": "", "transformers_version": "4.25.0.dev0", "n_ctx": 1024, "gradient_checkpointing": false, "model_type": "gpt2"} \ No newline at end of file diff --git a/perplexity25/transformers/pytorch_model.bin b/perplexity25/transformers/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..f9fc6b67d508381294a8b1759ea8b2625b83556e --- /dev/null +++ b/perplexity25/transformers/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78b702778fbf5e7de7d7219e1f3b1ed47195690d6ac1cd43d3b9517cd7da1467 +size 8781203669 diff --git a/perplexity50/3510899.err b/perplexity50/3510899.err new file mode 100644 index 0000000000000000000000000000000000000000..23081808f9a18dfd88dff6a5b90f74a58f654fd7 --- /dev/null +++ b/perplexity50/3510899.err @@ -0,0 +1,3635 @@ +20: 2023-05-13 09:35:56.410483: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:56.410501: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:56.410528: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:56.410567: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:56.410565: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:56.410584: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: 2023-05-13 09:35:56.410934: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:56.410586: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-13 09:35:56.410591: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:56.411006: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:56.411018: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:56.411027: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:56.411038: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:56.411052: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:56.411065: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-13 09:35:56.411085: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:56.411351: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:56.411359: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:56.411337: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:56.411370: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:56.411383: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: 2023-05-13 09:35:56.411438: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:56.411451: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:56.411465: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:56.411438: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:56.411393: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:56.411409: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-13 09:35:56.411450: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:56.411425: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:56.411443: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-13 09:35:56.411452: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: 2023-05-13 09:35:56.411489: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:56.411890: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:56.411899: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:56.411930: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:56.411972: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:56.411984: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:56.411992: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:56.412011: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-13 09:35:56.412027: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:56.655921: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:56.655931: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:56.656010: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:56.656029: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:56.656728: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:56.656741: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:56.656779: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:56.656849: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:56.656857: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:56.656871: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: 2023-05-13 09:35:56.656825: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:56.660882: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:56.664003: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:56.664449: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:56.664461: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:56.664463: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:56.664820: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:56.665775: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:56.666183: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:56.666199: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:56.666212: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:56.666724: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:56.666729: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:56.666725: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:56.667432: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:56.667445: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:56.667462: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:56.667470: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:56.668223: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:56.668225: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:56.668231: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:56.668225: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:56.668246: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:56.668931: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:56.668944: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:56.668936: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:56.669179: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:56.669189: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:56.669175: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:56.669175: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:56.669207: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:56.669460: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:56.669476: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:56.669481: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:56.669478: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:56.669496: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:56.669498: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:56.669854: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:56.669877: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:56.670233: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:56.670229: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:56.670248: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: 2023-05-13 09:35:56.670259: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:56.670274: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:56.670290: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:56.670292: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:56.671002: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:56.671427: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:56.672651: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:56.672752: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:56.676094: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:56.677793: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:56.677998: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:56.678784: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:56.679340: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:56.679809: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:56.680843: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:56.681532: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:56.682113: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:56.682470: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:56.683991: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:56.684873: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:56.686020: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:56.686743: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:56.686774: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:56.687712: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:56.687958: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:56.688286: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:56.688907: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:56.689142: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:56.689489: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:56.690913: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:56.692325: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:56.693670: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:56.695047: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:56.695277: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:56.696493: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:56.697171: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:56.698147: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:56.699056: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-13 09:35:56.700875: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-13 09:35:56.701099: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:56.702375: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:56.704034: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:56.704532: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:56.705124: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-13 09:35:56.705338: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:56.705945: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:56.706628: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:56.707625: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:56.707624: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:56.707631: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:56.707644: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:56.707643: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:56.707650: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:56.707639: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-13 09:35:56.707640: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: 2023-05-13 09:35:56.707859: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:56.707969: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:56.708551: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:56.708810: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:56.710508: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:56.712323: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:56.713260: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:56.716119: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-13 09:35:56.717148: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:56.717771: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:56.718208: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:56.718972: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:56.719158: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:56.719161: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:56.719174: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:56.719158: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:56.719178: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:56.719163: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:56.719183: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:35:56.719183: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:56.720781: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-13 09:35:56.721249: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:56.721809: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-13 09:35:56.723744: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:56.724218: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:56.724985: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-13 09:35:56.725680: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:56.726842: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-13 09:35:56.727400: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-13 09:35:56.728026: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-13 09:35:56.728601: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-13 09:35:56.729542: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-13 09:35:56.731424: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-13 09:35:56.732718: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-13 09:35:56.732894: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-13 09:35:56.735079: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-13 09:35:56.737674: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:56.783368: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:56.783379: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:56.783378: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:56.783390: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:56.783401: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:56.783401: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:56.783409: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-13 09:35:56.783412: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:56.783589: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:56.783596: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:56.783607: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:56.783623: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:56.783637: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:56.783640: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:56.783629: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-13 09:35:56.783648: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: 2023-05-13 09:35:56.783720: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:56.783729: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:56.783732: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:56.783725: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:56.783734: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:56.783744: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:56.783765: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:56.783803: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:56.783810: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:56.783844: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:56.783753: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:56.783764: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:56.783767: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:56.783782: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:56.783782: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:56.783858: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:56.783859: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:56.783861: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:56.783871: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:56.783879: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: 2023-05-13 09:35:56.783764: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:56.783771: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-13 09:35:56.783766: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:56.783859: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:56.783870: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:56.783909: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-13 09:35:56.783782: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:56.783860: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-13 09:35:56.783870: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:56.783913: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-13 09:35:56.783917: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:56.783937: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:56.783942: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:56.783956: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: 2023-05-13 09:35:56.784006: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:56.784017: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:56.784018: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: 2023-05-13 09:35:56.783902: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:56.783938: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:56.783967: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:56.784025: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:56.784013: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:56.783972: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:56.783983: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-13 09:35:56.783994: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: 2023-05-13 09:35:56.784039: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:56.784053: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-13 09:35:56.784060: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:56.784056: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:56.784058: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:56.784069: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:56.784077: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:56.784081: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:56.784071: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:56.784127: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-13 09:35:56.784127: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-13 09:36:09.902273: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:09.902302: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:09.902366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:09.902366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:09.902379: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:09.902420: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:09.902395: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:09.902385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:09.903336: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:36:09.903374: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:36:09.903398: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:36:09.903421: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:36:09.903427: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:36:09.903435: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:36:09.903453: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-13 09:36:09.903457: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:36:09.903061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:09.903092: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:09.903129: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:09.903122: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:09.903147: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:09.903167: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:09.903157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:09.903166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:09.903703: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:36:09.903718: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:36:09.903733: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:36:09.903748: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:36:09.903752: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:36:09.903756: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:36:09.903759: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-13 09:36:09.903771: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:36:09.903552: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:09.903583: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:09.903598: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:09.903633: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:09.903613: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:09.903651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:09.903636: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:09.903625: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:09.904127: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:36:09.904151: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:36:09.903826: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:09.904162: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:36:09.904193: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:36:09.904203: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:36:09.904205: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-13 09:36:09.904220: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:36:09.903850: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:09.903878: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:09.904257: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:36:09.903884: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:09.903891: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:09.903871: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:09.903907: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:09.903898: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:09.904378: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:36:09.904398: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:36:09.904413: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:36:09.904419: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:36:09.904075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:09.903957: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:09.904435: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:36:09.904430: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:36:09.904444: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-13 09:36:09.904449: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:36:09.904100: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:09.904135: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:09.904093: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:09.903978: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:09.904137: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:09.904114: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:09.903989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:09.904149: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:09.904116: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:09.904043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:09.904160: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:09.904122: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:09.904043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:09.904169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:09.904136: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:09.904035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:09.904168: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:09.904136: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:09.904075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:09.904194: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:09.904132: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:09.904079: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:09.904673: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:36:09.904623: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:36:09.904631: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:36:09.904622: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:36:09.904650: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:36:09.904652: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:36:09.904641: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:36:09.904644: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:36:09.904665: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:36:09.904669: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:36:09.904671: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:36:09.904670: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:36:09.904677: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:36:09.904681: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-13 09:36:09.904680: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:36:09.904693: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-13 09:36:09.904706: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:36:09.904698: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:36:09.904704: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:36:09.904717: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:36:09.904725: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:36:09.904732: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:36:09.904735: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-13 09:36:09.904754: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:36:09.904508: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:09.904535: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:09.904562: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:09.904585: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:09.904597: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:09.904607: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:09.904614: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:09.904635: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:09.905157: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:36:09.905175: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:36:09.905188: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:36:09.905206: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:36:09.905211: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:36:09.905208: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:36:09.905218: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-13 09:36:09.905221: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:36:09.910846: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:09.910871: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:09.910918: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:09.910910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:09.910894: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:09.910949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:09.910936: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:09.910974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:09.911588: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:36:09.911608: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:36:09.911619: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:36:09.911637: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:36:09.911644: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:36:09.911654: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:36:09.911658: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-13 09:36:09.911673: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:36:09.920038: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:09.920066: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:09.920086: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:09.920095: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:09.920116: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:09.920112: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:09.920121: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:09.920127: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:09.920633: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:36:09.920644: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:36:09.920649: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:36:09.920654: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:36:09.920667: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:36:09.920671: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:36:09.920673: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:36:09.920675: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:36:09.920571: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:09.920589: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:09.920604: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:09.920608: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:09.920616: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:09.920629: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:09.920620: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:09.920627: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:09.921072: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:36:09.921091: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:36:09.921102: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:36:09.921110: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:36:09.921118: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:36:09.921123: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:36:09.921132: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-13 09:36:09.921135: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:36:09.947162: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:09.947193: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:09.947183: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:09.947214: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:09.947402: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:09.947218: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:09.947222: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:09.947446: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:09.947416: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:09.947229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:09.947464: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:09.947431: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:09.947523: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:09.947238: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:09.947476: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:09.947471: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:09.947835: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:36:09.947492: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:09.947455: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:09.947855: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:36:09.947494: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:09.947461: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:09.947542: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:09.947867: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:36:09.947483: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:09.947476: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:09.947562: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:09.947499: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:09.947472: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:09.947573: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:09.947511: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:09.947944: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:36:09.947583: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:09.947894: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:36:09.947895: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:36:09.947901: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-13 09:36:09.947906: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:36:09.948009: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:36:09.947961: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:36:09.947597: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:09.947912: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:36:09.948026: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:36:09.947975: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:36:09.947590: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:09.948039: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:36:09.947985: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:36:09.947604: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:09.948054: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:36:09.948058: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:36:09.948000: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:36:09.948061: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:36:09.948064: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:36:09.948085: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:36:09.948093: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:36:09.948097: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:36:09.948095: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:36:09.948115: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:36:09.948016: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:36:09.948017: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-13 09:36:09.948021: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:36:09.948074: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-13 09:36:09.948075: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:36:09.948122: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-13 09:36:09.948134: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:36:09.947780: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:09.947791: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:09.947769: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:09.947827: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:09.947838: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:09.947997: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:09.947850: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:09.947985: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:09.947839: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:09.948014: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:09.947850: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:09.948035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:09.948038: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:09.947968: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:09.947988: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:09.948436: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:36:09.948453: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:36:09.947999: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:09.948009: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:09.948467: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:36:09.948022: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:09.948473: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:36:09.948479: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:36:09.948015: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:09.948487: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-13 09:36:09.948493: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:36:09.948092: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:09.948025: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:09.948495: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:36:09.948012: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:09.948527: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:36:09.948111: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:09.948547: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:36:09.948126: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:09.948559: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:36:09.948558: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:36:09.948149: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:09.948575: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:36:09.948152: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:09.948141: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:09.948342: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:09.948158: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:09.948582: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-13 09:36:09.948584: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:36:09.948156: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:09.948588: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:36:09.948358: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:09.948618: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:36:09.948373: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:09.948638: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:36:09.948648: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:36:09.948660: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:36:09.948664: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-13 09:36:09.948672: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:36:09.948392: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:09.948679: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:36:09.948394: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:09.948680: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:36:09.948401: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:09.948412: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:09.948410: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:09.948871: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:36:09.948889: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:36:09.948891: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:36:09.948913: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:36:09.948922: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:36:09.948925: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:36:09.948930: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-13 09:36:09.948936: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:36:09.948832: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:09.948848: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:09.948868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:09.948890: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:09.948897: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:09.948870: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:09.948878: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:09.948901: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:09.948904: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:09.948890: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:09.949342: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:36:09.948905: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:09.948904: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:09.949359: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:36:09.948915: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:09.949372: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:36:09.949377: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:36:09.948923: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:09.948949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:09.949387: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:36:09.948952: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:09.949404: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:36:09.949427: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:36:09.949434: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:36:09.949444: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:36:09.949402: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:36:09.949407: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-13 09:36:09.949400: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:36:09.949179: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:09.949452: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:36:09.949460: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-13 09:36:09.949464: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:36:09.949199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:09.949231: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:09.949245: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:09.949246: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:09.949220: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:09.949481: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:36:09.949259: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:09.949257: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:09.949686: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:36:09.949715: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:36:09.949717: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:36:09.949734: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:36:09.949742: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:36:09.949379: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:09.949746: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-13 09:36:09.949751: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:36:09.949404: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:09.949782: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:36:09.949411: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:09.949387: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:09.949424: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:09.949463: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:09.949429: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:09.949406: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:09.949439: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:09.949426: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:09.949461: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:09.949444: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:09.949443: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:09.949498: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:09.949917: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:36:09.949413: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:09.949463: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:09.949937: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:36:09.949452: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:09.949457: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:09.949516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:09.949945: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:36:09.949483: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:09.949524: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:09.949956: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:36:09.949502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:09.949533: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:09.949923: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:36:09.949497: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:09.949543: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:09.949963: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:36:09.949940: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:36:09.949516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:09.949552: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:09.949972: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:36:09.949524: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:09.949560: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:09.949978: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-13 09:36:09.949973: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:36:09.949531: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:09.949555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:09.949510: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:09.950060: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:36:09.949960: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:36:09.949954: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:36:09.949982: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:36:09.949983: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:36:09.949987: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:36:09.950046: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:36:09.950056: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:36:09.950075: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:36:09.950075: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:36:09.950071: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:36:09.950088: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:36:09.950092: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:36:09.950096: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:36:09.949688: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:09.950091: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:36:09.950092: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-13 09:36:09.950102: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:36:09.950101: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-13 09:36:09.949990: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:36:09.950111: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:36:09.949705: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:09.949709: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:09.949721: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:09.950117: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-13 09:36:09.950116: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:36:09.949734: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:09.949731: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:09.949740: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:09.949730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:09.950209: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:36:09.950236: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:36:09.950240: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:36:09.950246: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:36:09.950249: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:36:09.950251: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:36:09.950256: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-13 09:36:09.950255: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:36:09.949900: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:09.949917: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:09.949935: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:09.949944: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:09.949954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:09.949955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:09.949956: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:09.949964: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:09.950415: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:36:09.949932: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:09.949948: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:09.949967: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:09.949976: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:09.950436: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:36:09.950445: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:36:09.950454: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:36:09.950461: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:36:09.950465: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:36:09.949981: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:09.949968: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:09.950474: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-13 09:36:09.950473: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:36:09.949986: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:09.949993: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:09.950557: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:36:09.950574: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:36:09.950588: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:36:09.950594: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:36:09.950600: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:36:09.950604: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:36:09.950612: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-13 09:36:09.950614: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:36:09.951349: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:09.951366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:09.951378: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:09.951418: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:09.951450: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:09.951443: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:09.951473: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:09.951451: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:09.952051: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:36:09.952071: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:36:09.952078: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:36:09.952092: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:36:09.952098: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:36:09.952105: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:36:09.952108: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-13 09:36:09.952109: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:36:09.955386: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:09.955419: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:09.955448: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:09.955476: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:09.955515: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:09.955490: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:09.955453: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:09.955516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:09.956108: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:36:09.956113: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:36:09.956115: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:36:09.956124: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:36:09.956124: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:36:09.956131: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:36:09.956142: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-13 09:36:09.956149: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:36:09.948061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:09.948042: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:09.948057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:09.948493: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:36:09.948510: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:36:09.948525: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:36:09.948536: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:36:09.948544: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:36:09.948547: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:36:09.948548: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-13 09:36:09.948565: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-13 09:36:39.683332: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.683362: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.683388: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.683408: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.683422: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.683428: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.683434: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.683440: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.691962: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.691957: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.691998: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.691984: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.692017: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.691986: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.692031: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.691991: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.692044: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.692006: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.692048: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.692013: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.692060: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.692019: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.692067: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.692075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.693620: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.693649: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.693672: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.693753: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.693684: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.693796: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.693707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.693815: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.693708: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.693841: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.693725: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.693873: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.693730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.693879: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.693903: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.693921: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.694111: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.694171: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.694141: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.694200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.694164: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.694211: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.694177: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.694221: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.694186: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.694229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.694193: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.694248: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.694198: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.694246: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.694211: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.694259: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.694706: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.694724: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.694728: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.694743: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.694762: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.694763: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.694763: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.694772: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.695407: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.695438: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.695461: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.695476: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.695616: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.695497: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.695507: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.695504: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.695645: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.695525: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.695650: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.695671: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.695685: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.695697: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.695705: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.695716: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.696052: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.696096: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.696105: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.696144: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.696143: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.696157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.696169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.696195: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.696665: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.696704: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.696737: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.696777: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.696790: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.696819: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.696827: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.696847: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.697482: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.697511: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.697536: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.697547: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.697564: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.697560: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.697577: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.697720: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.697629: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.697654: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.697669: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.697670: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.697695: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.697697: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.697704: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.697874: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.698485: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.698506: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.698543: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.698556: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.698563: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.698569: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.698575: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.698584: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.698569: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.698602: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.698775: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.698627: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.698622: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.698643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.698647: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.698660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.698663: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.698815: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.698824: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.698825: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.698838: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.698858: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.698868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.698888: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.698859: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.698917: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.698880: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.698891: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.698896: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.698901: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.698938: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.698904: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.698944: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.699005: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.698957: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.699007: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.698966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.698966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.698977: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.698977: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.699112: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.699109: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.699143: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.699178: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.699189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.699205: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.699141: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.699210: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.699385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.699162: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.699222: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.699179: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.699224: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.699297: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.699198: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.699401: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.699199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.699413: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.699228: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.699423: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.699268: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.699429: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.699323: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.699438: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.699325: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.699436: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.699343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.699449: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.699345: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.699355: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.699358: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.699366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.699663: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.699679: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.699705: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.699714: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.699829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.699730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.699715: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.699722: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.699736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.699852: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.699877: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.699896: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.699903: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.699908: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.699910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.699965: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.703949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.703949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.703952: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.703959: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.703960: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.703961: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.703959: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.703960: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-13 09:36:39.703986: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:39.703988: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:39.703986: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:39.703987: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:39.703990: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:39.703992: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:39.703994: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-13 09:36:39.703994: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:39.704754: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.704651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.704748: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.704603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.704762: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.704651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.704754: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.704605: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.704761: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.704654: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.704755: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.704610: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.704767: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.704761: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.704652: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.704760: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.704613: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.704766: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.704779: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:39.704658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.704761: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.704611: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.704768: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.704657: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.704764: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.704614: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.704770: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.704772: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.704655: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.704766: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.704618: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.704773: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.704901: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.704775: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.704789: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:39.704661: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.704695: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:39.704763: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-13 09:36:39.704785: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:39.704623: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-13 09:36:39.704639: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:39.704774: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.704789: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:39.704796: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:39.704797: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:39.704695: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:39.704696: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:39.704698: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:39.704786: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:39.704786: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:39.704797: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:39.704640: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:39.704638: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:39.704642: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:39.704776: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.704903: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-13 09:36:39.704800: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:39.704802: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-13 09:36:39.704802: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:39.704700: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:39.704701: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-13 09:36:39.704703: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:39.704798: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:39.704799: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:39.704801: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:39.704647: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:39.704651: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:39.704649: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:39.704778: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.704906: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-13 09:36:39.704706: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-13 09:36:39.704802: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-13 09:36:39.704652: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:39.704779: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.704907: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.704778: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.704803: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:39.704910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.704803: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:39.704805: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:39.704805: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:39.704916: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.704807: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:39.704809: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-13 09:36:39.704807: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:39.704910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-13 09:36:39.704810: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:39.704917: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.704936: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:39.704935: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:39.704938: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:39.704938: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:39.705020: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-13 09:36:39.704943: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:39.704945: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:39.704947: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-13 09:36:39.704949: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:39.705022: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.705023: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.705029: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.705028: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.705030: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.705029: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.705033: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-13 09:36:39.705052: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:39.705052: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:39.705053: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:39.705054: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:39.705054: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:39.705055: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:39.705056: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-13 09:36:39.705055: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:39.705310: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.705311: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.705313: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.705315: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.705316: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.705466: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.705314: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.705322: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.705321: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-13 09:36:39.705351: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:39.705352: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:39.705353: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:39.705353: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:39.705355: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:39.705359: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:39.705360: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:39.705478: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.705491: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-13 09:36:39.705357: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:39.705480: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.705484: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.705496: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:39.705481: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.705486: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.705484: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.705481: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-13 09:36:39.705505: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:39.705507: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:39.705510: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:39.705512: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:39.705510: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-13 09:36:39.705509: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:39.705993: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.706020: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.706015: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.706032: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.706037: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.706039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.706051: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.706071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.706481: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.706483: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.706484: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.706484: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.706491: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.706490: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.706491: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.706493: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-13 09:36:39.706501: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:39.706501: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:39.706501: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:39.706504: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:39.706509: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:39.706511: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:39.706512: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-13 09:36:39.706511: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:39.707211: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.707260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.707234: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.707253: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.707259: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.707271: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.707260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.707278: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.707265: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.707288: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.707265: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.707291: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.707265: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.707269: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.707272: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.707272: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-13 09:36:39.707280: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:39.707280: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:39.707282: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:39.707283: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:39.707283: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:39.707284: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:39.707289: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-13 09:36:39.707289: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:39.707512: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.707516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.707516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.707519: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.707526: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:39.707520: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.707522: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.707524: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.707526: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-13 09:36:39.707537: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:39.707537: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:39.707538: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:39.707540: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:39.707540: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:39.707543: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-13 09:36:39.707544: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:39.707978: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.707979: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.707983: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.707985: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.707986: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.707985: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.707988: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.707984: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-13 09:36:39.708011: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:39.708011: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:39.708018: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:39.708020: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:39.708027: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:39.708030: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:39.708027: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-13 09:36:39.708032: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:39.708502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.708502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.708506: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.708508: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.708514: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.708505: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.708519: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.708541: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:39.708542: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:39.708544: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:39.708545: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:39.708550: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:39.708547: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:39.708546: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-13 09:36:39.708540: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-13 09:36:39.708579: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:39.708797: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.708797: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.708802: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.708805: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.708808: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.708802: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.708807: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.708807: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-13 09:36:39.708839: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:39.708839: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:39.708838: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:39.708842: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:39.708841: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:39.708846: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:39.708847: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-13 09:36:39.708843: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:39.709122: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.709130: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.709132: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.709136: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.709133: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.709140: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.709140: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.709139: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-13 09:36:39.709159: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:39.709161: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:39.709161: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:39.709166: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:39.709169: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:39.709174: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:39.709176: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-13 09:36:39.709177: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:39.709520: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.709539: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:39.709545: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.709554: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.709571: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:39.709577: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:39.709568: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.709569: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.709568: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.709574: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.709579: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-13 09:36:39.709599: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:39.709603: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:39.709605: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:39.709607: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-13 09:36:39.709608: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:39.709843: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.709853: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.709858: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:39.709855: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.709869: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:39.709866: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.709866: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.709870: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.709879: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:39.709873: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.709878: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-13 09:36:39.709890: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:39.709893: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:39.709894: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:39.709897: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-13 09:36:39.709903: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:39.710003: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.710120: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.710138: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:39.710009: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.710014: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.710028: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:39.710018: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.710033: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:39.710138: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.710018: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.710139: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.710162: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:39.710023: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.710163: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:39.710155: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.710105: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.710028: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.710165: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.710029: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.710054: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:39.710170: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.710055: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:39.710058: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:39.710064: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:39.710170: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-13 09:36:39.710065: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-13 09:36:39.710066: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:39.710178: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-13 09:36:39.710191: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:39.710195: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:39.710198: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:39.710199: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-13 09:36:39.710202: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:39.710103: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.710103: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.710324: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.710275: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.710109: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.710107: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.710112: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.710113: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.710335: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.710344: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:39.710278: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.710119: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.710139: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:39.710342: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.710279: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.710140: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:39.710147: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:39.710140: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:39.710342: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.710279: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.710149: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:39.710149: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-13 09:36:39.710143: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:39.710346: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.710279: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-13 09:36:39.710140: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:39.710343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.710282: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.710350: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.710361: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:39.710281: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.710290: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:39.710356: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-13 09:36:39.710367: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:39.710295: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:39.710300: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:39.710299: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:39.710370: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:39.710373: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:39.710375: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:39.710302: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:39.710302: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:39.710301: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:39.710374: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-13 09:36:39.710296: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-13 09:36:39.710324: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-13 09:36:39.710377: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:39.710544: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.710546: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.710546: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.710547: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.710551: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.710553: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.710554: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.710557: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-13 09:36:39.710567: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:39.710566: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:39.710566: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:39.710567: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:39.710568: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:39.710567: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:39.710570: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-13 09:36:39.710574: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:39.711168: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.711180: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.711187: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.711189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.711182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.711199: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:39.711192: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.711205: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.711220: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:39.711223: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:39.711224: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:39.711222: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:39.711211: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.711213: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.711226: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:39.711216: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-13 09:36:39.711232: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-13 09:36:39.711246: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:39.711220: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.711223: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.711224: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.711223: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.711221: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.711234: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-13 09:36:39.711248: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:39.711247: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:39.711256: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:39.711258: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:39.711257: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:39.711259: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:39.711262: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-13 09:36:39.711262: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:39.724225: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.724262: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.724277: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.724294: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.724320: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.724324: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.724328: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.724575: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.725568: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.725583: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:39.725578: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.725579: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.725582: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.725582: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.725584: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.725582: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.725588: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-13 09:36:39.725605: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:39.725605: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:39.725606: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:39.725607: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:39.725608: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:39.725610: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-13 09:36:39.725610: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:39.726097: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.726127: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.726139: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.726161: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.726187: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.726189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.726192: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.726385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.727343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.727379: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.727393: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.727415: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.727423: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.727437: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.727440: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.727447: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.727949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.727950: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.727952: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.727953: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.727955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:39.728007: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.727957: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.727958: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.727960: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.727971: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:39.727971: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:39.727973: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:39.727972: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:39.727974: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:39.727974: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-13 09:36:39.727976: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:39.728044: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-13 09:36:39.727976: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:39.728056: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:39.728093: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:39.728098: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:39.728102: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:39.728288: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:39.728290: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.728288: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.728306: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.728340: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.728357: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.728526: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.728525: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.728543: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.728735: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.728737: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.728739: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.728741: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.728742: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.728741: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.728743: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.728746: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-13 09:36:39.728752: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:39.728753: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:39.728755: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:39.728756: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:39.728758: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:39.728760: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:39.728758: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-13 09:36:39.728761: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:39.729537: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.728636: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.729549: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:39.729535: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.729555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:39.729540: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.729553: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:39.729544: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.729554: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:39.729545: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.729554: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.729565: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:39.729542: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.729556: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:39.729540: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-13 09:36:39.729553: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:39.729561: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.729569: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:39.729557: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:39.729561: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:39.729564: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:39.729564: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.729575: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:39.729565: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:39.729566: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:39.729568: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:39.729577: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:39.729579: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:39.729576: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:39.729631: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-13 09:36:39.729578: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-13 09:36:39.729582: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-13 09:36:39.729645: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: Loading extension module scaled_upper_triang_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module scaled_masked_softmax_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module scaled_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module fused_mix_prec_layer_norm_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module fused_mix_prec_layer_norm_cuda... +12: Successfully preprocessed all matching files. + 6: Successfully preprocessed all matching files. +27: Successfully preprocessed all matching files. +27: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. +27: Successfully preprocessed all matching files. +25: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. + 6: Successfully preprocessed all matching files. + 3: Successfully preprocessed all matching files. +26: Successfully preprocessed all matching files. +25: Successfully preprocessed all matching files. +25: Successfully preprocessed all matching files. +19: Successfully preprocessed all matching files. + 7: Successfully preprocessed all matching files. +10: Successfully preprocessed all matching files. +10: Successfully preprocessed all matching files. +10: Successfully preprocessed all matching files. +29: Successfully preprocessed all matching files. +29: Successfully preprocessed all matching files. + 5: Successfully preprocessed all matching files. +11: Successfully preprocessed all matching files. +18: Successfully preprocessed all matching files. + 2: Successfully preprocessed all matching files. + 2: Successfully preprocessed all matching files. + 2: Successfully preprocessed all matching files. +12: Successfully preprocessed all matching files. + 4: Successfully preprocessed all matching files. +13: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: + 0: + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 4: + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: + 7: +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: +13: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: + 8: + 8: + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: + 9: + 9: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: +14: +14: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +15: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: +16: +16: +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +17: +17: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +21: +21: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +20: +20: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: +23: +23: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +22: +22: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: +24: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +28: +28: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: +30: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... + 2: Building extension module utils... + 2: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +12: +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 3: Loading extension module utils... + 0: Loading extension module utils... + 1: Loading extension module utils... + 3: Loading extension module utils... + 0: Loading extension module utils... + 1: Loading extension module utils... + 3: Loading extension module utils... + 0: Loading extension module utils... + 3: Loading extension module utils... + 1: Loading extension module utils... + 1: Loading extension module utils... +10: Loading extension module utils... +10: Loading extension module utils... +10: Loading extension module utils... +10: Loading extension module utils... + 4: Loading extension module utils... +11: Loading extension module utils... + 4: Loading extension module utils... +11: Loading extension module utils... + 4: Loading extension module utils... +11: Loading extension module utils... + 4: Loading extension module utils... +11: Loading extension module utils... + 8: Loading extension module utils... + 7: Loading extension module utils... +12: Loading extension module utils... + 7: Loading extension module utils... + 8: Loading extension module utils... +12: Loading extension module utils... + 6: Loading extension module utils... +12: Loading extension module utils... +13: Loading extension module utils... + 7: Loading extension module utils... + 8: Loading extension module utils... +12: Loading extension module utils... + 7: Loading extension module utils... +13: Loading extension module utils... + 8: Loading extension module utils... + 6: Loading extension module utils... +13: Loading extension module utils... + 6: Loading extension module utils... + 6: Loading extension module utils... +13: Loading extension module utils... + 9: Loading extension module utils... + 9: Loading extension module utils... + 9: Loading extension module utils... + 9: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... + 5: Loading extension module utils... +25: Loading extension module utils... +29: Loading extension module utils... +14: Loading extension module utils... +26: Loading extension module utils... +17: Loading extension module utils... +21: Loading extension module utils... +16: Loading extension module utils... +19: Loading extension module utils... + 5: Loading extension module utils... +25: Loading extension module utils... +27: Loading extension module utils... +26: Loading extension module utils... + 5: Loading extension module utils... +17: Loading extension module utils... +21: Loading extension module utils... +16: Loading extension module utils... +29: Loading extension module utils... +15: Loading extension module utils... +20: Loading extension module utils... +14: Loading extension module utils... +23: Loading extension module utils... +31: Loading extension module utils... +19: Loading extension module utils... +25: Loading extension module utils... +21: Loading extension module utils... +22: Loading extension module utils... +14: Loading extension module utils... + 5: Loading extension module utils... +27: Loading extension module utils... +17: Loading extension module utils... +25: Loading extension module utils... +29: Loading extension module utils... +26: Loading extension module utils... +14: Loading extension module utils... +20: Loading extension module utils... +21: Loading extension module utils... +16: Loading extension module utils... +23: Loading extension module utils... +31: Loading extension module utils... +15: Loading extension module utils... +22: Loading extension module utils... +29: Loading extension module utils... +17: Loading extension module utils... +19: Loading extension module utils... +28: Loading extension module utils... +27: Loading extension module utils... +15: Loading extension module utils... +24: Loading extension module utils... +20: Loading extension module utils... +16: Loading extension module utils... +22: Loading extension module utils... +26: Loading extension module utils... +23: Loading extension module utils... +31: Loading extension module utils... +27: Loading extension module utils... +28: Loading extension module utils... +15: Loading extension module utils... +19: Loading extension module utils... +24: Loading extension module utils... +23: Loading extension module utils... +22: Loading extension module utils... +20: Loading extension module utils... +31: Loading extension module utils... +28: Loading extension module utils... +24: Loading extension module utils... +28: Loading extension module utils... +30: Loading extension module utils... +24: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 1: Loading extension module utils... + 1: Loading extension module utils... + 1: Loading extension module utils... + 2: Loading extension module utils... + 1: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 3: Loading extension module utils... + 2: Loading extension module utils... + 3: Loading extension module utils... + 6: Loading extension module utils... + 4: Loading extension module utils... + 3: Loading extension module utils... + 4: Loading extension module utils... + 3: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... + 6: Loading extension module utils... + 6: Loading extension module utils... + 7: Loading extension module utils... + 6: Loading extension module utils... + 7: Loading extension module utils... + 7: Loading extension module utils... + 7: Loading extension module utils... +10: Loading extension module utils... +10: Loading extension module utils... + 8: Loading extension module utils... + 8: Loading extension module utils... +10: Loading extension module utils... +11: Loading extension module utils... + 8: Loading extension module utils... + 9: Loading extension module utils... + 9: Loading extension module utils... +11: Loading extension module utils... + 8: Loading extension module utils... + 9: Loading extension module utils... + 9: Loading extension module utils... +10: Loading extension module utils... +13: Loading extension module utils... +12: Loading extension module utils... +11: Loading extension module utils... +12: Loading extension module utils... +13: Loading extension module utils... +11: Loading extension module utils... +12: Loading extension module utils... +12: Loading extension module utils... +13: Loading extension module utils... +13: Loading extension module utils... +14: Loading extension module utils... +14: Loading extension module utils... +14: Loading extension module utils... +14: Loading extension module utils... +15: Loading extension module utils... +15: Loading extension module utils... +16: Loading extension module utils... +15: Loading extension module utils... +17: Loading extension module utils... +16: Loading extension module utils... +15: Loading extension module utils... +16: Loading extension module utils... +17: Loading extension module utils... +18: Loading extension module utils... +17: Loading extension module utils... +16: Loading extension module utils... + 5: Loading extension module utils... +18: Loading extension module utils... +17: Loading extension module utils... + 5: Loading extension module utils... + 5: Loading extension module utils... +19: Loading extension module utils... +18: Loading extension module utils... +19: Loading extension module utils... +18: Loading extension module utils... + 5: Loading extension module utils... +19: Loading extension module utils... +19: Loading extension module utils... +21: Loading extension module utils... +20: Loading extension module utils... +21: Loading extension module utils... +20: Loading extension module utils... +20: Loading extension module utils... +20: Loading extension module utils... +21: Loading extension module utils... +23: Loading extension module utils... +22: Loading extension module utils... +23: Loading extension module utils... +23: Loading extension module utils... +22: Loading extension module utils... +25: Loading extension module utils... +22: Loading extension module utils... +23: Loading extension module utils... +26: Loading extension module utils... +22: Loading extension module utils... +25: Loading extension module utils... +24: Loading extension module utils... +25: Loading extension module utils... +24: Loading extension module utils... +26: Loading extension module utils... +24: Loading extension module utils... +24: Loading extension module utils... +25: Loading extension module utils... +26: Loading extension module utils... +26: Loading extension module utils... +27: Loading extension module utils... +27: Loading extension module utils... +27: Loading extension module utils... +27: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +29: Loading extension module utils... +29: Loading extension module utils... +28: Loading extension module utils... +30: Loading extension module utils... +29: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +29: Loading extension module utils... +31: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... +21: Loading extension module utils... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +29: + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: + 6: + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Loading extension module utils...Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +29: +29: +29: Loading extension module utils...Loading extension module utils... +29: +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +29: + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 6: + 6: + 6: + 6: Loading extension module utils...Loading extension module utils...Loading extension module utils... + 6: + 6: + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 1: + 1: Loading extension module utils... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +12: +12: Loading extension module utils... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +12: +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: Loading extension module utils... +27: Loading extension module utils... +27: +27: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +27: +27: Loading extension module utils...Loading extension module utils... +27: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 3: + 3: + 3: + 3: + 3: Loading extension module utils...Loading extension module utils...Loading extension module utils... + 3: + 3: Loading extension module utils...Loading extension module utils... + 3: + 3: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 5: + 5: + 5: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 5: + 5: + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +21: +21: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +21: +21: +21: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +21: +21: +21: Loading extension module utils...Loading extension module utils... +21: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: +21: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +21: +21: Loading extension module utils... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 2: + 2: Loading extension module utils...Loading extension module utils... + 2: + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +18: +18: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: +18: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +18: +18: Loading extension module utils... +18: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +11: +11: Loading extension module utils...Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +11: +11: +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +11: +11: Loading extension module utils...Loading extension module utils... +11: +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +18: +18: +18: Loading extension module utils...Loading extension module utils...Loading extension module utils... +18: +18: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +31: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +31: +31: Loading extension module utils... +31: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +31: +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +31: Loading extension module utils... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +15: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +15: +15: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +10: +10: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +15: +15: Loading extension module utils...Loading extension module utils... +15: +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +19: +19: Loading extension module utils... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +19: +19: Loading extension module utils... +19: Loading extension module utils... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +22: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +30: +30: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +30: +30: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +22: +22: +22: Loading extension module utils...Loading extension module utils...Loading extension module utils... +22: +22: +22: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +22: +22: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: +22: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +22: +22: Loading extension module utils... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +20: +30: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +30: +30: Loading extension module utils...Loading extension module utils... +30: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... +20: +20: +20: +20: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... +20: +20: +20: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +25: +25: Loading extension module utils...Loading extension module utils... +25: +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: +20: Loading extension module utils...Loading extension module utils...Loading extension module utils... +20: +20: +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +26: +26: Loading extension module utils...Loading extension module utils... +26: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +17: +17: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...No modifications detected for re-loaded extension module utils, skipping build step... +17: +17: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +17: +17: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... +17: +17: +17: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +17: +17: +17: Loading extension module utils...Loading extension module utils... +17: +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: + 7: + 7: + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 7: + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... + 7: + 7: + 7: Loading extension module utils... + 7: + 7: Loading extension module utils...Loading extension module utils... + 7: + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +28: +28: +28: +28: Loading extension module utils...Loading extension module utils...Loading extension module utils...Loading extension module utils... +28: +28: +28: +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +14: +14: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +14: +14: +14: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 4: + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 4: + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 4: + 4: Loading extension module utils... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 4: + 4: Loading extension module utils... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +24: +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +24: +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +13: +13: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... +13: +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: + 9: + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 9: + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: + 8: + 8: + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 8: + 8: Loading extension module utils...Loading extension module utils... + 8: + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 8: + 8: Loading extension module utils...Loading extension module utils... + 8: +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +16: +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: + 0: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 0: + 0: + 0: Loading extension module utils...Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: + 0: + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +26: +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +26: +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: +23: +23: +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +23: +23: +23: Loading extension module utils...Loading extension module utils...Loading extension module utils... +23: +23: +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings + 0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/perplexity50/3510899.out b/perplexity50/3510899.out new file mode 100644 index 0000000000000000000000000000000000000000..03e1eb801bfff0652088545d6c44c3843690de21 --- /dev/null +++ b/perplexity50/3510899.out @@ -0,0 +1,42548 @@ +Model parameters: d_model 3072 ffw_size 12288 kv_size 128 n_heads 24 n_layers 36 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 2 --pipeline-model-parallel-size 1 --num-layers 36 --hidden-size 3072 --num-attention-heads 24 --kv-channels 128 --ffn-hidden-size 12288 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 2 --global-batch-size 1024 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-4b284bc4perplexity50val --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --override-lr-scheduler --reset-progress --no-load-optim --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --eval-only true --tensorboard-dir tensorboard_4b284bc4perplexity50val --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_4b284bc4perplexity50 --load checkpoints_4b284bc4perplexity50 --train-weighted-split-paths-path train1b5.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/3510899.json --zero-stage 0 +START 3510899: Sat 13 May 2023 09:35:06 AM EEST + 0: + 0: + 0: ======================= ROCm System Management Interface ======================= + 0: ================================= Concise Info ================================= + 0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 0: 0 42.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 2 38.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 4 40.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 6 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: ================================================================================ + 0: ============================= End of ROCm SMI Log ============================== + 6: + 6: + 6: ======================= ROCm System Management Interface ======================= + 6: ================================= Concise Info ================================= + 6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 6: 0 44.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 2 39.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 4 44.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 6 47.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: ================================================================================ + 6: ============================= End of ROCm SMI Log ============================== + 7: + 7: + 7: ======================= ROCm System Management Interface ======================= + 7: ================================= Concise Info ================================= + 7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 7: 0 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 2 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 4 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 6 40.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: ================================================================================ + 7: ============================= End of ROCm SMI Log ============================== + 4: + 4: + 4: ======================= ROCm System Management Interface ======================= + 4: ================================= Concise Info ================================= + 4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 4: 0 46.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 2 35.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 4 40.0c 80.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 6 35.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: ================================================================================ + 4: ============================= End of ROCm SMI Log ============================== + 1: + 1: + 1: ======================= ROCm System Management Interface ======================= + 1: ================================= Concise Info ================================= + 1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 1: 0 52.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 2 38.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 4 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 6 38.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: ================================================================================ + 1: ============================= End of ROCm SMI Log ============================== +14: +14: +14: ======================= ROCm System Management Interface ======================= +14: ================================= Concise Info ================================= +14: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +14: 0 46.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 2 38.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 4 38.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 6 38.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: ================================================================================ +14: ============================= End of ROCm SMI Log ============================== +13: +13: +13: ======================= ROCm System Management Interface ======================= +13: ================================= Concise Info ================================= +13: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +13: 0 46.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 2 49.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 4 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 6 39.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: ================================================================================ +13: ============================= End of ROCm SMI Log ============================== +15: +15: +15: ======================= ROCm System Management Interface ======================= +15: ================================= Concise Info ================================= +15: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +15: 0 44.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 2 39.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 4 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 6 38.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: ================================================================================ +15: ============================= End of ROCm SMI Log ============================== +18: +18: +18: ======================= ROCm System Management Interface ======================= +18: ================================= Concise Info ================================= +18: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +18: 0 43.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 2 40.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 4 47.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 6 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: ================================================================================ +18: ============================= End of ROCm SMI Log ============================== +22: +22: +22: ======================= ROCm System Management Interface ======================= +22: ================================= Concise Info ================================= +22: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +22: 0 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 2 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 4 42.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 6 45.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: ================================================================================ +22: ============================= End of ROCm SMI Log ============================== +20: +20: +20: ======================= ROCm System Management Interface ======================= +20: ================================= Concise Info ================================= +20: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +20: 0 49.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 2 45.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 4 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 6 42.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: ================================================================================ +20: ============================= End of ROCm SMI Log ============================== +27: +27: +27: ======================= ROCm System Management Interface ======================= +27: ================================= Concise Info ================================= +27: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +27: 0 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 2 39.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 4 42.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 6 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: ================================================================================ +27: ============================= End of ROCm SMI Log ============================== +29: +29: +29: ======================= ROCm System Management Interface ======================= +29: ================================= Concise Info ================================= +29: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +29: 0 46.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 2 44.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 4 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 6 40.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: ================================================================================ +29: ============================= End of ROCm SMI Log ============================== +25: +25: +25: ======================= ROCm System Management Interface ======================= +25: ================================= Concise Info ================================= +25: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +25: 0 45.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 2 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 4 47.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 6 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: ================================================================================ +25: ============================= End of ROCm SMI Log ============================== +12: +12: +12: ======================= ROCm System Management Interface ======================= +12: ================================= Concise Info ================================= +12: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +12: 0 48.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 2 38.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 4 42.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 6 42.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: ================================================================================ +12: ============================= End of ROCm SMI Log ============================== + 8: + 8: + 8: ======================= ROCm System Management Interface ======================= + 8: ================================= Concise Info ================================= + 8: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 8: 0 46.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 2 39.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 4 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 6 45.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: ================================================================================ + 8: ============================= End of ROCm SMI Log ============================== + 5: + 5: + 5: ======================= ROCm System Management Interface ======================= + 5: ================================= Concise Info ================================= + 5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 5: 0 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 2 39.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 4 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 6 40.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 7 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: ================================================================================ + 5: ============================= End of ROCm SMI Log ============================== +16: +16: +16: ======================= ROCm System Management Interface ======================= +16: ================================= Concise Info ================================= +16: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +16: 0 49.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 2 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 4 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 6 42.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: ================================================================================ +16: ============================= End of ROCm SMI Log ============================== +21: +21: +21: ======================= ROCm System Management Interface ======================= +21: ================================= Concise Info ================================= +21: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +21: 0 48.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 2 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 4 44.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 6 40.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: ================================================================================ +21: ============================= End of ROCm SMI Log ============================== +24: +24: +24: ======================= ROCm System Management Interface ======================= +24: ================================= Concise Info ================================= +24: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +24: 0 46.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 2 42.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 4 44.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 6 35.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: ================================================================================ +24: ============================= End of ROCm SMI Log ============================== +28: +28: +28: ======================= ROCm System Management Interface ======================= +28: ================================= Concise Info ================================= +28: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +28: 0 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 2 41.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 4 45.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 6 35.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: ================================================================================ +28: ============================= End of ROCm SMI Log ============================== + 2: + 2: + 2: ======================= ROCm System Management Interface ======================= + 2: ================================= Concise Info ================================= + 2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 2: 0 45.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 2 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 4 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 6 45.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: ================================================================================ + 2: ============================= End of ROCm SMI Log ============================== +30: +30: +30: ======================= ROCm System Management Interface ======================= +30: ================================= Concise Info ================================= +30: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +30: 0 48.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 2 44.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 4 46.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 6 40.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: ================================================================================ +30: ============================= End of ROCm SMI Log ============================== + 3: + 3: + 3: ======================= ROCm System Management Interface ======================= + 3: ================================= Concise Info ================================= + 3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 3: 0 44.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 2 44.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 4 36.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 6 40.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: ================================================================================ + 3: ============================= End of ROCm SMI Log ============================== +10: +10: +10: ======================= ROCm System Management Interface ======================= +10: ================================= Concise Info ================================= +10: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +10: 0 46.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 2 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 4 40.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 6 43.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: ================================================================================ +10: ============================= End of ROCm SMI Log ============================== +11: +11: +11: ======================= ROCm System Management Interface ======================= +11: ================================= Concise Info ================================= +11: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +11: 0 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 2 36.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 4 43.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 6 39.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: ================================================================================ +11: ============================= End of ROCm SMI Log ============================== +17: +17: +17: ======================= ROCm System Management Interface ======================= +17: ================================= Concise Info ================================= +17: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +17: 0 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 2 40.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 4 38.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 5 52.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 6 38.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: ================================================================================ +17: ============================= End of ROCm SMI Log ============================== + 9: + 9: + 9: ======================= ROCm System Management Interface ======================= + 9: ================================= Concise Info ================================= + 9: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 9: 0 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 2 44.0c 79.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 4 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 6 42.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: ================================================================================ + 9: ============================= End of ROCm SMI Log ============================== +31: +31: +31: ======================= ROCm System Management Interface ======================= +31: ================================= Concise Info ================================= +31: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +31: 0 47.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 2 40.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 4 36.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 6 37.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: ================================================================================ +31: ============================= End of ROCm SMI Log ============================== +19: +19: +19: ======================= ROCm System Management Interface ======================= +19: ================================= Concise Info ================================= +19: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +19: 0 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 2 40.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 4 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 6 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: ================================================================================ +19: ============================= End of ROCm SMI Log ============================== +26: +26: +26: ======================= ROCm System Management Interface ======================= +26: ================================= Concise Info ================================= +26: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +26: 0 45.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 2 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 4 47.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 6 45.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: ================================================================================ +26: ============================= End of ROCm SMI Log ============================== +23: +23: +23: ======================= ROCm System Management Interface ======================= +23: ================================= Concise Info ================================= +23: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +23: 0 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 2 42.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 4 40.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 6 40.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: ================================================================================ +23: ============================= End of ROCm SMI Log ============================== +27: Launching on nid006668 (27/32), master nid005918 port 9999, GPUs 8, CUDA: True +13: Launching on nid005931 (13/32), master nid005918 port 9999, GPUs 8, CUDA: True +25: Launching on nid006666 (25/32), master nid005918 port 9999, GPUs 8, CUDA: True + 5: Launching on nid005923 (5/32), master nid005918 port 9999, GPUs 8, CUDA: True +29: Launching on nid006670 (29/32), master nid005918 port 9999, GPUs 8, CUDA: True + 4: Launching on nid005922 (4/32), master nid005918 port 9999, GPUs 8, CUDA: True + 1: Launching on nid005919 (1/32), master nid005918 port 9999, GPUs 8, CUDA: True +18: Launching on nid006659 (18/32), master nid005918 port 9999, GPUs 8, CUDA: True + 7: Launching on nid005925 (7/32), master nid005918 port 9999, GPUs 8, CUDA: True +20: Launching on nid006661 (20/32), master nid005918 port 9999, GPUs 8, CUDA: True +14: Launching on nid005932 (14/32), master nid005918 port 9999, GPUs 8, CUDA: True + 6: Launching on nid005924 (6/32), master nid005918 port 9999, GPUs 8, CUDA: True + 0: Launching on nid005918 (0/32), master nid005918 port 9999, GPUs 8, CUDA: True +16: Launching on nid005934 (16/32), master nid005918 port 9999, GPUs 8, CUDA: True +30: Launching on nid006671 (30/32), master nid005918 port 9999, GPUs 8, CUDA: True +15: Launching on nid005933 (15/32), master nid005918 port 9999, GPUs 8, CUDA: True +22: Launching on nid006663 (22/32), master nid005918 port 9999, GPUs 8, CUDA: True +21: Launching on nid006662 (21/32), master nid005918 port 9999, GPUs 8, CUDA: True + 8: Launching on nid005926 (8/32), master nid005918 port 9999, GPUs 8, CUDA: True +28: Launching on nid006669 (28/32), master nid005918 port 9999, GPUs 8, CUDA: True +24: Launching on nid006665 (24/32), master nid005918 port 9999, GPUs 8, CUDA: True +12: Launching on nid005930 (12/32), master nid005918 port 9999, GPUs 8, CUDA: True + 2: Launching on nid005920 (2/32), master nid005918 port 9999, GPUs 8, CUDA: True +26: Launching on nid006667 (26/32), master nid005918 port 9999, GPUs 8, CUDA: True +17: Launching on nid006658 (17/32), master nid005918 port 9999, GPUs 8, CUDA: True + 9: Launching on nid005927 (9/32), master nid005918 port 9999, GPUs 8, CUDA: True +31: Launching on nid006672 (31/32), master nid005918 port 9999, GPUs 8, CUDA: True +19: Launching on nid006660 (19/32), master nid005918 port 9999, GPUs 8, CUDA: True +11: Launching on nid005929 (11/32), master nid005918 port 9999, GPUs 8, CUDA: True +23: Launching on nid006664 (23/32), master nid005918 port 9999, GPUs 8, CUDA: True +10: Launching on nid005928 (10/32), master nid005918 port 9999, GPUs 8, CUDA: True + 3: Launching on nid005921 (3/32), master nid005918 port 9999, GPUs 8, CUDA: True + 0: using world size: 256, data-parallel-size: 128, tensor-model-parallel size: 2, pipeline-model-parallel size: 1 + 0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. + 0: using torch.bfloat16 for parameters ... + 0: ------------------------ arguments ------------------------ + 0: abort_on_unmet_fused_kernel_constraints ......... False + 0: accumulate_allreduce_grads_in_fp32 .............. True + 0: adam_beta1 ...................................... 0.9 + 0: adam_beta2 ...................................... 0.999 + 0: adam_eps ........................................ 1e-08 + 0: adlr_autoresume ................................. False + 0: adlr_autoresume_interval ........................ 1000 + 0: apply_query_key_layer_scaling ................... True + 0: apply_residual_connection_post_layernorm ........ False + 0: attention_dropout ............................... 0.1 + 0: attention_softmax_in_fp32 ....................... False + 0: bert_binary_head ................................ True + 0: bert_load ....................................... None + 0: bf16 ............................................ True + 0: bias_dropout_fusion ............................. True + 0: bias_gelu_fusion ................................ True + 0: biencoder_projection_dim ........................ 0 + 0: biencoder_shared_query_context_model ............ False + 0: block_data_path ................................. None + 0: checkpoint_activations .......................... False + 0: checkpoint_in_cpu ............................... False + 0: checkpoint_num_layers ........................... 1 + 0: clip_grad ....................................... 1.0 + 0: codecarbon_dir .................................. None + 0: consumed_train_samples .......................... 0 + 0: consumed_train_tokens ........................... 0 + 0: consumed_valid_samples .......................... 0 + 0: contigious_checkpointing ........................ False + 0: cpu_optimizer ................................... False + 0: cpu_torch_adam .................................. False + 0: curriculum_learning ............................. False + 0: data_impl ....................................... mmap + 0: data_parallel_size .............................. 128 + 0: data_path ....................................... None + 0: dataloader_type ................................. single + 0: DDP_impl ........................................ local + 0: decoder_seq_length .............................. None + 0: deepscale ....................................... False + 0: deepscale_config ................................ None + 0: deepspeed ....................................... True + 0: deepspeed_activation_checkpointing .............. False + 0: deepspeed_config ................................ ds_configs/3510899.json + 0: deepspeed_mpi ................................... False + 0: distribute_checkpointed_activations ............. False + 0: distributed_backend ............................. nccl + 0: embed_layernorm ................................. False + 0: embedding_path .................................. None + 0: encoder_seq_length .............................. 2048 + 0: eod_mask_loss ................................... False + 0: eval_interval ................................... 1 + 0: eval_iters ...................................... 100 + 0: eval_only ....................................... True + 0: evidence_data_path .............................. None + 0: exit_duration_in_mins ........................... None + 0: exit_interval ................................... None + 0: ffn_hidden_size ................................. 12288 + 0: finetune ........................................ False + 0: fp16 ............................................ False + 0: fp16_lm_cross_entropy ........................... False + 0: fp32_residual_connection ........................ False + 0: gigaflos_no_embeds .............................. 0 + 0: global_batch_size ............................... 1024 + 0: glu_activation .................................. None + 0: hidden_dropout .................................. 0.1 + 0: hidden_size ..................................... 3072 + 0: hysteresis ...................................... 2 + 0: ict_head_size ................................... None + 0: ict_load ........................................ None + 0: img_dim ......................................... 224 + 0: indexer_batch_size .............................. 128 + 0: indexer_log_interval ............................ 1000 + 0: inference ....................................... False + 0: init_method_std ................................. 0.02 + 0: init_method_xavier_uniform ...................... False + 0: initial_loss_scale .............................. 4294967296 + 0: kill_switch_path ................................ kill-switch-4b284bc4perplexity50val + 0: kv_channels ..................................... 128 + 0: layer_norm_fusion ............................... True + 0: layernorm_epsilon ............................... 1e-05 + 0: lazy_mpu_init ................................... None + 0: load ............................................ checkpoints_4b284bc4perplexity50 + 0: local_rank ...................................... None + 0: log_batch_size_to_tensorboard ................... True + 0: log_interval .................................... 10 + 0: log_learning_rate_to_tensorboard ................ True + 0: log_level ....................................... None + 0: log_level_replica ............................... None + 0: log_loss_scale_to_tensorboard ................... True + 0: log_num_zeros_in_grad ........................... False + 0: log_params_norm ................................. False + 0: log_path ........................................ None + 0: log_timers_to_tensorboard ....................... True + 0: log_validation_ppl_to_tensorboard ............... True + 0: loss_on_targets_only ............................ False + 0: loss_scale ...................................... None + 0: loss_scale_window ............................... 1000 + 0: lr .............................................. 0.0002 + 0: lr_decay_iters .................................. None + 0: lr_decay_samples ................................ 1 + 0: lr_decay_style .................................. cosine + 0: lr_decay_tokens ................................. None + 0: lr_warmup_fraction .............................. None + 0: lr_warmup_iters ................................. 0 + 0: lr_warmup_samples ............................... 0 + 0: make_vocab_size_divisible_by .................... 128 + 0: mask_prob ....................................... 0.15 + 0: masked_softmax_fusion ........................... True + 0: max_position_embeddings ......................... 2048 + 0: mean_noise_span_length .......................... None + 0: memory_centric_tiled_linear ..................... False + 0: merge_file ...................................... gpt2/merges.txt + 0: micro_batch_size ................................ 2 + 0: min_loss_scale .................................. 1.0 + 0: min_lr .......................................... 2e-05 + 0: mmap_warmup ..................................... False + 0: no_load_optim ................................... True + 0: no_load_rng ..................................... None + 0: no_save_optim ................................... None + 0: no_save_rng ..................................... None + 0: noise_density ................................... None + 0: num_attention_heads ............................. 24 + 0: num_channels .................................... 3 + 0: num_classes ..................................... 1000 + 0: num_layers ...................................... 36 + 0: num_layers_per_virtual_pipeline_stage ........... None + 0: num_workers ..................................... 2 + 0: onnx_safe ....................................... None + 0: openai_gelu ..................................... False + 0: optimizer ....................................... adam + 0: optimizer_fusion ................................ True + 0: override_lr_scheduler ........................... True + 0: pad_vocab_size_to ............................... None + 0: params_dtype .................................... torch.bfloat16 + 0: partition_activations ........................... False + 0: patch_dim ....................................... 16 + 0: pipeline_model_parallel_size .................... 1 + 0: position_embedding_type ......................... PositionEmbeddingType.absolute + 0: pp_partition_method ............................. None + 0: profile_backward ................................ False + 0: query_in_block_prob ............................. 0.1 + 0: rampup_batch_size ............................... None + 0: rank ............................................ 0 + 0: remote_device ................................... none + 0: reset_attention_mask ............................ False + 0: reset_position_ids .............................. False + 0: reset_progress .................................. True + 0: retriever_report_topk_accuracies ................ [] + 0: retriever_score_scaling ......................... False + 0: retriever_seq_length ............................ 256 + 0: reweight_loss_based_on_position_frequency ....... False + 0: sample_rate ..................................... 1.0 + 0: save ............................................ checkpoints_4b284bc4perplexity50 + 0: save_interval ................................... 1000 + 0: scatter_gather_tensors_in_pipeline .............. True + 0: scattered_embeddings ............................ False + 0: seed ............................................ 1234 + 0: seq_length ...................................... 2048 + 0: sgd_momentum .................................... 0.9 + 0: short_seq_prob .................................. 0.1 + 0: skip_train_iteration_range ...................... None + 0: split ........................................... None + 0: split_transformers .............................. False + 0: sync_tp_duplicated_parameters ................... False + 0: synchronize_each_layer .......................... False + 0: tensor_model_parallel_size ...................... 2 + 0: tensorboard_dir ................................. tensorboard_4b284bc4perplexity50val + 0: tensorboard_log_interval ........................ 1 + 0: tensorboard_queue_size .......................... 5 + 0: test_weighted_split_paths ....................... None + 0: test_weighted_split_paths_path .................. None + 0: tile_factor ..................................... 1 + 0: titles_data_path ................................ None + 0: tokenizer_name_or_path .......................... None + 0: tokenizer_type .................................. GPT2BPETokenizer + 0: train_iters ..................................... None + 0: train_samples ................................... 1 + 0: train_tokens .................................... None + 0: train_weighted_split_names ...................... ['train'] + 0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document']] + 0: train_weighted_split_paths_path ................. None + 0: train_weighted_split_splits ..................... [['0:1']] + 0: train_weighted_split_weights .................... [['1.0']] + 0: universal_checkpoint ............................ False + 0: use_bnb_optimizer ............................... False + 0: use_checkpoint_lr_scheduler ..................... False + 0: use_contiguous_buffers_in_ddp ................... True + 0: use_cpu_initialization .......................... None + 0: use_one_sent_docs ............................... False + 0: use_pin_memory .................................. False + 0: valid_num_workers ............................... 2 + 0: valid_weighted_split_names ...................... ['validation'] + 0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] + 0: valid_weighted_split_paths_path ................. None + 0: valid_weighted_split_splits ..................... [['0:1']] + 0: valid_weighted_split_weights .................... [['1.0']] + 0: virtual_pipeline_model_parallel_size ............ None + 0: vocab_extra_ids ................................. 0 + 0: vocab_file ...................................... gpt2/vocab.json + 0: weight_decay .................................... 0.1 + 0: world_size ...................................... 256 + 0: zero_allgather_bucket_size ...................... 0.0 + 0: zero_contigious_gradients ....................... False + 0: zero_reduce_bucket_size ......................... 0.0 + 0: zero_reduce_scatter ............................. False + 0: zero_stage ...................................... 0 + 0: -------------------- end of arguments --------------------- + 0: setting number of micro-batches to constant 4 + 0: > building GPT2BPETokenizer tokenizer ... + 0: > padded vocab (size: 50257) with 175 dummy tokens (new size: 50432) + 0: DeepSpeed general environment info: + 0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] + 0: torch version .................... 1.13.0+rocm5.2 + 0: torch cuda version ............... None + 0: torch hip version ................ 5.2.21151-afdc89f8 + 0: nvcc version ..................... None + 0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] + 0: deepspeed info ................... 0.7.5, unknown, unknown + 0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 + 0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** + 0: > initializing torch distributed ... + 0: [2023-05-13 09:37:46,236] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +31: > setting tensorboard ... + 0: > initializing tensor model parallel with size 2 + 0: > initializing pipeline model parallel with size 1 + 0: > setting random seeds to 1234 ... + 0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 + 0: > compiling dataset index builder ... + 0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: make: Nothing to be done for 'default'. + 0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: >>> done with dataset index builder. Compilation time: 0.098 seconds + 0: > compiling and loading fused kernels ... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 102 + 0: ninja: no work to do. + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 67 + 0: ninja: no work to do. + 0: >>> done with compiling and loading fused kernels. Compilation time: 33.077 seconds + 0: time to initialize megatron (seconds): 100.770 + 0: [after megatron is initialized] datetime: 2023-05-13 09:38:28 + 0: building GPT model ... + 0: [2023-05-13 09:38:28,924] [INFO] [utils.py:827:see_memory_usage] Before Building Model + 0: [2023-05-13 09:38:28,929] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB + 0: [2023-05-13 09:38:28,930] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.7 GB, percent = 7.9% + 0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None + 0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=0, model=1): 1, ProcessCoord(pipe=0, data=1, model=0): 2, ProcessCoord(pipe=0, data=1, model=1): 3, ProcessCoord(pipe=0, data=2, model=0): 4, ProcessCoord(pipe=0, data=2, model=1): 5, ProcessCoord(pipe=0, data=3, model=0): 6, ProcessCoord(pipe=0, data=3, model=1): 7, ProcessCoord(pipe=0, data=4, model=0): 8, ProcessCoord(pipe=0, data=4, model=1): 9, ProcessCoord(pipe=0, data=5, model=0): 10, ProcessCoord(pipe=0, data=5, model=1): 11, ProcessCoord(pipe=0, data=6, model=0): 12, ProcessCoord(pipe=0, data=6, model=1): 13, ProcessCoord(pipe=0, data=7, model=0): 14, ProcessCoord(pipe=0, data=7, model=1): 15, ProcessCoord(pipe=0, data=8, model=0): 16, ProcessCoord(pipe=0, data=8, model=1): 17, ProcessCoord(pipe=0, data=9, model=0): 18, ProcessCoord(pipe=0, data=9, model=1): 19, ProcessCoord(pipe=0, data=10, model=0): 20, ProcessCoord(pipe=0, data=10, model=1): 21, ProcessCoord(pipe=0, data=11, model=0): 22, ProcessCoord(pipe=0, data + 0: =11, model=1): 23, ProcessCoord(pipe=0, data=12, model=0): 24, ProcessCoord(pipe=0, data=12, model=1): 25, ProcessCoord(pipe=0, data=13, model=0): 26, ProcessCoord(pipe=0, data=13, model=1): 27, ProcessCoord(pipe=0, data=14, model=0): 28, ProcessCoord(pipe=0, data=14, model=1): 29, ProcessCoord(pipe=0, data=15, model=0): 30, ProcessCoord(pipe=0, data=15, model=1): 31, ProcessCoord(pipe=0, data=16, model=0): 32, ProcessCoord(pipe=0, data=16, model=1): 33, ProcessCoord(pipe=0, data=17, model=0): 34, ProcessCoord(pipe=0, data=17, model=1): 35, ProcessCoord(pipe=0, data=18, model=0): 36, ProcessCoord(pipe=0, data=18, model=1): 37, ProcessCoord(pipe=0, data=19, model=0): 38, ProcessCoord(pipe=0, data=19, model=1): 39, ProcessCoord(pipe=0, data=20, model=0): 40, ProcessCoord(pipe=0, data=20, model=1): 41, ProcessCoord(pipe=0, data=21, model=0): 42, ProcessCoord(pipe=0, data=21, model=1): 43, ProcessCoord(pipe=0, data=22, model=0): 44, ProcessCoord(pipe=0, data=22, model=1): 45, ProcessCoord(pipe=0, data=23, model=0 + 0: ): 46, ProcessCoord(pipe=0, data=23, model=1): 47, ProcessCoord(pipe=0, data=24, model=0): 48, ProcessCoord(pipe=0, data=24, model=1): 49, ProcessCoord(pipe=0, data=25, model=0): 50, ProcessCoord(pipe=0, data=25, model=1): 51, ProcessCoord(pipe=0, data=26, model=0): 52, ProcessCoord(pipe=0, data=26, model=1): 53, ProcessCoord(pipe=0, data=27, model=0): 54, ProcessCoord(pipe=0, data=27, model=1): 55, ProcessCoord(pipe=0, data=28, model=0): 56, ProcessCoord(pipe=0, data=28, model=1): 57, ProcessCoord(pipe=0, data=29, model=0): 58, ProcessCoord(pipe=0, data=29, model=1): 59, ProcessCoord(pipe=0, data=30, model=0): 60, ProcessCoord(pipe=0, data=30, model=1): 61, ProcessCoord(pipe=0, data=31, model=0): 62, ProcessCoord(pipe=0, data=31, model=1): 63, ProcessCoord(pipe=0, data=32, model=0): 64, ProcessCoord(pipe=0, data=32, model=1): 65, ProcessCoord(pipe=0, data=33, model=0): 66, ProcessCoord(pipe=0, data=33, model=1): 67, ProcessCoord(pipe=0, data=34, model=0): 68, ProcessCoord(pipe=0, data=34, model=1): 69, Proce + 0: ssCoord(pipe=0, data=35, model=0): 70, ProcessCoord(pipe=0, data=35, model=1): 71, ProcessCoord(pipe=0, data=36, model=0): 72, ProcessCoord(pipe=0, data=36, model=1): 73, ProcessCoord(pipe=0, data=37, model=0): 74, ProcessCoord(pipe=0, data=37, model=1): 75, ProcessCoord(pipe=0, data=38, model=0): 76, ProcessCoord(pipe=0, data=38, model=1): 77, ProcessCoord(pipe=0, data=39, model=0): 78, ProcessCoord(pipe=0, data=39, model=1): 79, ProcessCoord(pipe=0, data=40, model=0): 80, ProcessCoord(pipe=0, data=40, model=1): 81, ProcessCoord(pipe=0, data=41, model=0): 82, ProcessCoord(pipe=0, data=41, model=1): 83, ProcessCoord(pipe=0, data=42, model=0): 84, ProcessCoord(pipe=0, data=42, model=1): 85, ProcessCoord(pipe=0, data=43, model=0): 86, ProcessCoord(pipe=0, data=43, model=1): 87, ProcessCoord(pipe=0, data=44, model=0): 88, ProcessCoord(pipe=0, data=44, model=1): 89, ProcessCoord(pipe=0, data=45, model=0): 90, ProcessCoord(pipe=0, data=45, model=1): 91, ProcessCoord(pipe=0, data=46, model=0): 92, ProcessCoord(pipe + 0: =0, data=46, model=1): 93, ProcessCoord(pipe=0, data=47, model=0): 94, ProcessCoord(pipe=0, data=47, model=1): 95, ProcessCoord(pipe=0, data=48, model=0): 96, ProcessCoord(pipe=0, data=48, model=1): 97, ProcessCoord(pipe=0, data=49, model=0): 98, ProcessCoord(pipe=0, data=49, model=1): 99, ProcessCoord(pipe=0, data=50, model=0): 100, ProcessCoord(pipe=0, data=50, model=1): 101, ProcessCoord(pipe=0, data=51, model=0): 102, ProcessCoord(pipe=0, data=51, model=1): 103, ProcessCoord(pipe=0, data=52, model=0): 104, ProcessCoord(pipe=0, data=52, model=1): 105, ProcessCoord(pipe=0, data=53, model=0): 106, ProcessCoord(pipe=0, data=53, model=1): 107, ProcessCoord(pipe=0, data=54, model=0): 108, ProcessCoord(pipe=0, data=54, model=1): 109, ProcessCoord(pipe=0, data=55, model=0): 110, ProcessCoord(pipe=0, data=55, model=1): 111, ProcessCoord(pipe=0, data=56, model=0): 112, ProcessCoord(pipe=0, data=56, model=1): 113, ProcessCoord(pipe=0, data=57, model=0): 114, ProcessCoord(pipe=0, data=57, model=1): 115, ProcessCoord( + 0: pipe=0, data=58, model=0): 116, ProcessCoord(pipe=0, data=58, model=1): 117, ProcessCoord(pipe=0, data=59, model=0): 118, ProcessCoord(pipe=0, data=59, model=1): 119, ProcessCoord(pipe=0, data=60, model=0): 120, ProcessCoord(pipe=0, data=60, model=1): 121, ProcessCoord(pipe=0, data=61, model=0): 122, ProcessCoord(pipe=0, data=61, model=1): 123, ProcessCoord(pipe=0, data=62, model=0): 124, ProcessCoord(pipe=0, data=62, model=1): 125, ProcessCoord(pipe=0, data=63, model=0): 126, ProcessCoord(pipe=0, data=63, model=1): 127, ProcessCoord(pipe=0, data=64, model=0): 128, ProcessCoord(pipe=0, data=64, model=1): 129, ProcessCoord(pipe=0, data=65, model=0): 130, ProcessCoord(pipe=0, data=65, model=1): 131, ProcessCoord(pipe=0, data=66, model=0): 132, ProcessCoord(pipe=0, data=66, model=1): 133, ProcessCoord(pipe=0, data=67, model=0): 134, ProcessCoord(pipe=0, data=67, model=1): 135, ProcessCoord(pipe=0, data=68, model=0): 136, ProcessCoord(pipe=0, data=68, model=1): 137, ProcessCoord(pipe=0, data=69, model=0): 138, Pr + 0: ocessCoord(pipe=0, data=69, model=1): 139, ProcessCoord(pipe=0, data=70, model=0): 140, ProcessCoord(pipe=0, data=70, model=1): 141, ProcessCoord(pipe=0, data=71, model=0): 142, ProcessCoord(pipe=0, data=71, model=1): 143, ProcessCoord(pipe=0, data=72, model=0): 144, ProcessCoord(pipe=0, data=72, model=1): 145, ProcessCoord(pipe=0, data=73, model=0): 146, ProcessCoord(pipe=0, data=73, model=1): 147, ProcessCoord(pipe=0, data=74, model=0): 148, ProcessCoord(pipe=0, data=74, model=1): 149, ProcessCoord(pipe=0, data=75, model=0): 150, ProcessCoord(pipe=0, data=75, model=1): 151, ProcessCoord(pipe=0, data=76, model=0): 152, ProcessCoord(pipe=0, data=76, model=1): 153, ProcessCoord(pipe=0, data=77, model=0): 154, ProcessCoord(pipe=0, data=77, model=1): 155, ProcessCoord(pipe=0, data=78, model=0): 156, ProcessCoord(pipe=0, data=78, model=1): 157, ProcessCoord(pipe=0, data=79, model=0): 158, ProcessCoord(pipe=0, data=79, model=1): 159, ProcessCoord(pipe=0, data=80, model=0): 160, ProcessCoord(pipe=0, data=80, model= + 0: 1): 161, ProcessCoord(pipe=0, data=81, model=0): 162, ProcessCoord(pipe=0, data=81, model=1): 163, ProcessCoord(pipe=0, data=82, model=0): 164, ProcessCoord(pipe=0, data=82, model=1): 165, ProcessCoord(pipe=0, data=83, model=0): 166, ProcessCoord(pipe=0, data=83, model=1): 167, ProcessCoord(pipe=0, data=84, model=0): 168, ProcessCoord(pipe=0, data=84, model=1): 169, ProcessCoord(pipe=0, data=85, model=0): 170, ProcessCoord(pipe=0, data=85, model=1): 171, ProcessCoord(pipe=0, data=86, model=0): 172, ProcessCoord(pipe=0, data=86, model=1): 173, ProcessCoord(pipe=0, data=87, model=0): 174, ProcessCoord(pipe=0, data=87, model=1): 175, ProcessCoord(pipe=0, data=88, model=0): 176, ProcessCoord(pipe=0, data=88, model=1): 177, ProcessCoord(pipe=0, data=89, model=0): 178, ProcessCoord(pipe=0, data=89, model=1): 179, ProcessCoord(pipe=0, data=90, model=0): 180, ProcessCoord(pipe=0, data=90, model=1): 181, ProcessCoord(pipe=0, data=91, model=0): 182, ProcessCoord(pipe=0, data=91, model=1): 183, ProcessCoord(pipe=0, data + 0: =92, model=0): 184, ProcessCoord(pipe=0, data=92, model=1): 185, ProcessCoord(pipe=0, data=93, model=0): 186, ProcessCoord(pipe=0, data=93, model=1): 187, ProcessCoord(pipe=0, data=94, model=0): 188, ProcessCoord(pipe=0, data=94, model=1): 189, ProcessCoord(pipe=0, data=95, model=0): 190, ProcessCoord(pipe=0, data=95, model=1): 191, ProcessCoord(pipe=0, data=96, model=0): 192, ProcessCoord(pipe=0, data=96, model=1): 193, ProcessCoord(pipe=0, data=97, model=0): 194, ProcessCoord(pipe=0, data=97, model=1): 195, ProcessCoord(pipe=0, data=98, model=0): 196, ProcessCoord(pipe=0, data=98, model=1): 197, ProcessCoord(pipe=0, data=99, model=0): 198, ProcessCoord(pipe=0, data=99, model=1): 199, ProcessCoord(pipe=0, data=100, model=0): 200, ProcessCoord(pipe=0, data=100, model=1): 201, ProcessCoord(pipe=0, data=101, model=0): 202, ProcessCoord(pipe=0, data=101, model=1): 203, ProcessCoord(pipe=0, data=102, model=0): 204, ProcessCoord(pipe=0, data=102, model=1): 205, ProcessCoord(pipe=0, data=103, model=0): 206, Process + 0: Coord(pipe=0, data=103, model=1): 207, ProcessCoord(pipe=0, data=104, model=0): 208, ProcessCoord(pipe=0, data=104, model=1): 209, ProcessCoord(pipe=0, data=105, model=0): 210, ProcessCoord(pipe=0, data=105, model=1): 211, ProcessCoord(pipe=0, data=106, model=0): 212, ProcessCoord(pipe=0, data=106, model=1): 213, ProcessCoord(pipe=0, data=107, model=0): 214, ProcessCoord(pipe=0, data=107, model=1): 215, ProcessCoord(pipe=0, data=108, model=0): 216, ProcessCoord(pipe=0, data=108, model=1): 217, ProcessCoord(pipe=0, data=109, model=0): 218, ProcessCoord(pipe=0, data=109, model=1): 219, ProcessCoord(pipe=0, data=110, model=0): 220, ProcessCoord(pipe=0, data=110, model=1): 221, ProcessCoord(pipe=0, data=111, model=0): 222, ProcessCoord(pipe=0, data=111, model=1): 223, ProcessCoord(pipe=0, data=112, model=0): 224, ProcessCoord(pipe=0, data=112, model=1): 225, ProcessCoord(pipe=0, data=113, model=0): 226, ProcessCoord(pipe=0, data=113, model=1): 227, ProcessCoord(pipe=0, data=114, model=0): 228, ProcessCoord(pipe=0 + 0: , data=114, model=1): 229, ProcessCoord(pipe=0, data=115, model=0): 230, ProcessCoord(pipe=0, data=115, model=1): 231, ProcessCoord(pipe=0, data=116, model=0): 232, ProcessCoord(pipe=0, data=116, model=1): 233, ProcessCoord(pipe=0, data=117, model=0): 234, ProcessCoord(pipe=0, data=117, model=1): 235, ProcessCoord(pipe=0, data=118, model=0): 236, ProcessCoord(pipe=0, data=118, model=1): 237, ProcessCoord(pipe=0, data=119, model=0): 238, ProcessCoord(pipe=0, data=119, model=1): 239, ProcessCoord(pipe=0, data=120, model=0): 240, ProcessCoord(pipe=0, data=120, model=1): 241, ProcessCoord(pipe=0, data=121, model=0): 242, ProcessCoord(pipe=0, data=121, model=1): 243, ProcessCoord(pipe=0, data=122, model=0): 244, ProcessCoord(pipe=0, data=122, model=1): 245, ProcessCoord(pipe=0, data=123, model=0): 246, ProcessCoord(pipe=0, data=123, model=1): 247, ProcessCoord(pipe=0, data=124, model=0): 248, ProcessCoord(pipe=0, data=124, model=1): 249, ProcessCoord(pipe=0, data=125, model=0): 250, ProcessCoord(pipe=0, data=125, + 0: model=1): 251, ProcessCoord(pipe=0, data=126, model=0): 252, ProcessCoord(pipe=0, data=126, model=1): 253, ProcessCoord(pipe=0, data=127, model=0): 254, ProcessCoord(pipe=0, data=127, model=1): 255} + 0: [2023-05-13 09:38:34,610] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer + 0: stage=0 layers=43 + 0: 0: _to_float16 + 0: 1: EmbeddingPipe + 0: 2: + 0: 3: ParallelTransformerLayerPipe + 0: 4: ParallelTransformerLayerPipe + 0: 5: ParallelTransformerLayerPipe + 0: 6: ParallelTransformerLayerPipe + 0: 7: ParallelTransformerLayerPipe + 0: 8: ParallelTransformerLayerPipe + 0: 9: ParallelTransformerLayerPipe + 0: 10: ParallelTransformerLayerPipe + 0: 11: ParallelTransformerLayerPipe + 0: 12: ParallelTransformerLayerPipe + 0: 13: ParallelTransformerLayerPipe + 0: 14: ParallelTransformerLayerPipe + 0: 15: ParallelTransformerLayerPipe + 0: 16: ParallelTransformerLayerPipe + 0: 17: ParallelTransformerLayerPipe + 0: 18: ParallelTransformerLayerPipe + 0: 19: ParallelTransformerLayerPipe + 0: 20: ParallelTransformerLayerPipe + 0: 21: ParallelTransformerLayerPipe + 0: 22: ParallelTransformerLayerPipe + 0: 23: ParallelTransformerLayerPipe + 0: 24: ParallelTransformerLayerPipe + 0: 25: ParallelTransformerLayerPipe + 0: 26: ParallelTransformerLayerPipe + 0: 27: ParallelTransformerLayerPipe + 0: 28: ParallelTransformerLayerPipe + 0: 29: ParallelTransformerLayerPipe + 0: 30: ParallelTransformerLayerPipe + 0: 31: ParallelTransformerLayerPipe + 0: 32: ParallelTransformerLayerPipe + 0: 33: ParallelTransformerLayerPipe + 0: 34: ParallelTransformerLayerPipe + 0: 35: ParallelTransformerLayerPipe + 0: 36: ParallelTransformerLayerPipe + 0: 37: ParallelTransformerLayerPipe + 0: 38: ParallelTransformerLayerPipe + 0: 39: undo + 0: 40: MixedFusedLayerNorm + 0: 41: EmbeddingPipe + 0: 42: float16_to_fp32 + 0: loss: CrossEntropy + 0: [2023-05-13 09:38:34,971] [INFO] [utils.py:827:see_memory_usage] After Building Model + 0: [2023-05-13 09:38:34,972] [INFO] [utils.py:828:see_memory_usage] MA 4.0 GB Max_MA 4.0 GB CA 4.04 GB Max_CA 4 GB + 0: [2023-05-13 09:38:34,972] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.95 GB, percent = 7.9% + 0: setting training iterations to 0 + 0: > learning rate decay style: cosine + 0: DeepSpeed is enabled. + 0: [2023-05-13 09:38:34,975] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown + 0: [2023-05-13 09:38:41,124] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False + 0: [2023-05-13 09:38:41,124] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer + 0: [2023-05-13 09:38:41,124] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer + 0: [2023-05-13 09:38:41,146] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam + 0: [2023-05-13 09:38:41,146] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer + 0: [2023-05-13 09:38:41,269] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer + 0: [2023-05-13 09:38:41,270] [INFO] [utils.py:828:see_memory_usage] MA 3.99 GB Max_MA 4.01 GB CA 4.05 GB Max_CA 4 GB + 0: [2023-05-13 09:38:41,270] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.59 GB, percent = 8.1% + 2: ninja: no work to do. +29: Time to load utils op: 0.14753127098083496 secondsTime to load utils op: 0.1475691795349121 secondsTime to load utils op: 0.14777588844299316 secondsTime to load utils op: 0.14664554595947266 seconds +29: +29: +29: +29: Time to load utils op: 0.2781662940979004 seconds +29: Time to load utils op: 0.27898335456848145 secondsTime to load utils op: 0.27905774116516113 seconds +29: +29: Time to load utils op: 0.2794930934906006 seconds + 6: Time to load utils op: 0.16021490097045898 seconds + 6: Time to load utils op: 0.16277647018432617 seconds + 6: Time to load utils op: 0.16164326667785645 secondsTime to load utils op: 0.2922663688659668 seconds + 6: + 6: Time to load utils op: 0.2925872802734375 seconds + 6: Time to load utils op: 0.29239511489868164 secondsTime to load utils op: 0.16048908233642578 secondsTime to load utils op: 0.292161226272583 seconds + 6: + 6: +13: Time to load utils op: 0.18767595291137695 seconds +13: Time to load utils op: 0.1882190704345703 secondsTime to load utils op: 0.3193178176879883 seconds +13: +13: Time to load utils op: 0.18825435638427734 secondsTime to load utils op: 0.18947386741638184 secondsTime to load utils op: 0.3196554183959961 seconds +13: +13: +13: Time to load utils op: 0.3196711540222168 seconds +13: Time to load utils op: 0.31955885887145996 seconds +12: Time to load utils op: 0.2077312469482422 seconds +12: Time to load utils op: 0.2077775001525879 seconds +12: Time to load utils op: 0.20760869979858398 secondsTime to load utils op: 0.2078104019165039 seconds +12: +12: Time to load utils op: 0.33922362327575684 seconds +25: Time to load utils op: 0.20002150535583496 seconds +12: Time to load utils op: 0.3374500274658203 seconds +12: Time to load utils op: 0.33881449699401855 secondsTime to load utils op: 0.33883118629455566 seconds +12: +25: Time to load utils op: 0.33214330673217773 secondsTime to load utils op: 0.20107626914978027 secondsTime to load utils op: 0.19998717308044434 seconds +25: +25: +25: Time to load utils op: 0.3322296142578125 secondsTime to load utils op: 0.20096969604492188 seconds +25: +25: Time to load utils op: 0.3319888114929199 seconds +25: Time to load utils op: 0.33184003829956055 seconds +18: Time to load utils op: 0.20515918731689453 secondsTime to load utils op: 0.20445489883422852 secondsTime to load utils op: 0.20427274703979492 seconds +18: Time to load utils op: 0.3346550464630127 seconds +18: +18: Time to load utils op: 0.20540738105773926 seconds +18: +18: Time to load utils op: 0.33487820625305176 secondsTime to load utils op: 0.3349335193634033 secondsTime to load utils op: 0.33450937271118164 seconds +18: +18: +16: Time to load utils op: 0.20537710189819336 secondsTime to load utils op: 0.20551276206970215 secondsTime to load utils op: 0.2056107521057129 seconds +16: +16: +16: Time to load utils op: 0.3326585292816162 seconds +16: Time to load utils op: 0.3326606750488281 secondsTime to load utils op: 0.20554852485656738 seconds +16: +16: Time to load utils op: 0.3326733112335205 secondsTime to load utils op: 0.3326742649078369 seconds +16: +20: Time to load utils op: 0.20306777954101562 seconds +20: Time to load utils op: 0.2030797004699707 seconds +20: Time to load utils op: 0.20316576957702637 seconds +10: Time to load utils op: 0.2088916301727295 seconds +20: Time to load utils op: 0.3300354480743408 seconds +20: Time to load utils op: 0.330045223236084 seconds +20: Time to load utils op: 0.3300514221191406 secondsTime to load utils op: 0.20310449600219727 seconds +20: +20: Time to load utils op: 0.33007025718688965 seconds +10: Time to load utils op: 0.21041107177734375 seconds +10: Time to load utils op: 0.20988082885742188 seconds +10: Time to load utils op: 0.34099316596984863 seconds +10: Time to load utils op: 0.21005797386169434 secondsTime to load utils op: 0.340437650680542 seconds +10: Time to load utils op: 0.33235907554626465 seconds +10: Time to load utils op: 0.3395805358886719 seconds +10: + 5: Time to load utils op: 0.20501399040222168 secondsTime to load utils op: 0.20467066764831543 seconds + 5: + 5: Time to load utils op: 0.20551538467407227 seconds + 5: Time to load utils op: 0.20565462112426758 seconds + 5: Time to load utils op: 0.32771849632263184 secondsTime to load utils op: 0.3278462886810303 seconds + 5: + 5: Time to load utils op: 0.3278932571411133 seconds + 5: Time to load utils op: 0.3307042121887207 seconds +11: Time to load utils op: 0.2084660530090332 seconds +11: Time to load utils op: 0.21018171310424805 seconds +11: Time to load utils op: 0.21080636978149414 seconds +11: Time to load utils op: 0.3397843837738037 secondsTime to load utils op: 0.20872974395751953 secondsTime to load utils op: 0.34052085876464844 seconds +11: +11: +11: Time to load utils op: 0.33975768089294434 seconds +11: Time to load utils op: 0.3399472236633301 seconds + 1: Time to load utils op: 0.21575260162353516 seconds + 1: Time to load utils op: 0.21577692031860352 secondsTime to load utils op: 0.2154684066772461 seconds + 1: + 1: Time to load utils op: 0.34595799446105957 seconds + 1: Time to load utils op: 0.34571051597595215 secondsTime to load utils op: 0.21521615982055664 seconds + 1: + 4: Time to load utils op: 0.21355938911437988 secondsTime to load utils op: 0.2143876552581787 secondsTime to load utils op: 0.21432781219482422 secondsTime to load utils op: 0.21371173858642578 seconds + 4: + 4: + 4: + 1: Time to load utils op: 0.34595322608947754 seconds + 1: Time to load utils op: 0.3455524444580078 seconds +28: Time to load utils op: 0.1995394229888916 secondsTime to load utils op: 0.19958043098449707 secondsTime to load utils op: 0.19968390464782715 seconds +28: +28: +28: Time to load utils op: 0.326763391494751 seconds + 4: Time to load utils op: 0.34368133544921875 seconds + 4: Time to load utils op: 0.343860387802124 seconds + 4: Time to load utils op: 0.34352946281433105 seconds + 4: Time to load utils op: 0.34377241134643555 seconds +28: Time to load utils op: 0.32677268981933594 secondsTime to load utils op: 0.326765775680542 seconds +28: +28: Time to load utils op: 0.20043230056762695 seconds +28: Time to load utils op: 0.3267686367034912 seconds +27: Time to load utils op: 0.200850248336792 seconds +27: Time to load utils op: 0.20053505897521973 secondsTime to load utils op: 0.3317911624908447 seconds +27: +27: Time to load utils op: 0.20180821418762207 secondsTime to load utils op: 0.33197736740112305 secondsTime to load utils op: 0.20039868354797363 seconds +27: +27: +27: Time to load utils op: 0.3326570987701416 seconds +27: Time to load utils op: 0.33145976066589355 seconds +14: Time to load utils op: 0.20785856246948242 seconds +14: Time to load utils op: 0.20798087120056152 secondsTime to load utils op: 0.20815539360046387 seconds +14: +14: Time to load utils op: 0.33554625511169434 seconds +14: Time to load utils op: 0.33556127548217773 seconds +14: Time to load utils op: 0.2079944610595703 secondsTime to load utils op: 0.33542871475219727 seconds +14: +14: Time to load utils op: 0.3355977535247803 seconds +23: Time to load utils op: 0.20301532745361328 secondsTime to load utils op: 0.20318078994750977 secondsTime to load utils op: 0.3296787738800049 seconds +23: +23: +23: Time to load utils op: 0.20330595970153809 seconds +23: Time to load utils op: 0.20250368118286133 seconds +23: Time to load utils op: 0.3296947479248047 seconds +23: Time to load utils op: 0.32970285415649414 seconds +23: Time to load utils op: 0.32970643043518066 seconds +15: Time to load utils op: 0.20700716972351074 secondsTime to load utils op: 0.20704317092895508 seconds +15: +15: Time to load utils op: 0.207597017288208 seconds +15: Time to load utils op: 0.20732688903808594 seconds + 0: Time to load utils op: 0.2166125774383545 secondsTime to load utils op: 0.23390579223632812 seconds + 0: Time to load utils op: 0.2160177230834961 seconds + 0: + 0: Time to load utils op: 0.34708642959594727 secondsTime to load utils op: 0.24092555046081543 seconds + 0: + 0: Time to load utils op: 0.21604323387145996 seconds + 0: Time to load utils op: 0.3471074104309082 seconds +15: Time to load utils op: 0.3345816135406494 secondsTime to load utils op: 0.33497071266174316 seconds +15: +26: Time to load utils op: 0.20215845108032227 secondsTime to load utils op: 0.33262205123901367 seconds +26: +26: Time to load utils op: 0.20222163200378418 secondsTime to load utils op: 0.3326268196105957 secondsTime to load utils op: 0.2025318145751953 seconds +26: +26: +26: Time to load utils op: 0.33246827125549316 seconds +26: Time to load utils op: 0.3325843811035156 seconds + 0: Time to load utils op: 0.34702610969543457 seconds +15: Time to load utils op: 0.3350036144256592 seconds +26: Time to load utils op: 0.20358777046203613 seconds +15: Time to load utils op: 0.33502197265625 seconds +22: Time to load utils op: 0.20374298095703125 secondsTime to load utils op: 0.20377516746520996 secondsTime to load utils op: 0.20417475700378418 secondsTime to load utils op: 0.3301103115081787 seconds +22: +22: Time to load utils op: 0.3301079273223877 seconds +22: +22: Time to load utils op: 0.3301126956939697 secondsTime to load utils op: 0.33011913299560547 seconds +22: +22: +22: Time to load utils op: 0.20381522178649902 seconds + 2: Time to load utils op: 0.2165241241455078 seconds + 2: Time to load utils op: 0.21541118621826172 secondsTime to load utils op: 0.21569156646728516 secondsTime to load utils op: 0.21550250053405762 seconds + 2: + 2: + 2: Time to load utils op: 0.34776902198791504 seconds + 2: Time to load utils op: 0.34707069396972656 seconds + 2: Time to load utils op: 0.3463141918182373 seconds + 2: Time to load utils op: 0.34789347648620605 seconds +30: Time to load utils op: 0.20121192932128906 seconds +31: Time to load utils op: 0.33007216453552246 secondsTime to load utils op: 0.201124906539917 secondsTime to load utils op: 0.20086193084716797 seconds +31: +31: +31: Time to load utils op: 0.2003481388092041 seconds +31: Time to load utils op: 0.3321070671081543 secondsTime to load utils op: 0.20079326629638672 seconds +31: +30: Time to load utils op: 0.3266940116882324 secondsTime to load utils op: 0.2004537582397461 seconds +30: +30: Time to load utils op: 0.32669830322265625 seconds +30: Time to load utils op: 0.3266918659210205 secondsTime to load utils op: 0.3267087936401367 seconds +30: +31: Time to load utils op: 0.33191919326782227 secondsTime to load utils op: 0.33159399032592773 seconds +31: +30: Time to load utils op: 0.200439453125 seconds +30: Time to load utils op: 0.20050382614135742 seconds +17: Time to load utils op: 0.20799565315246582 seconds +17: Time to load utils op: 0.2088639736175537 seconds +17: Time to load utils op: 0.20824670791625977 seconds +17: Time to load utils op: 0.33527469635009766 secondsTime to load utils op: 0.335282564163208 seconds +17: +17: Time to load utils op: 0.20801544189453125 secondsTime to load utils op: 0.3352956771850586 seconds +17: +17: Time to load utils op: 0.335341215133667 seconds + 3: Time to load utils op: 0.2167670726776123 secondsTime to load utils op: 0.2167208194732666 seconds + 3: + 3: Time to load utils op: 0.2162339687347412 seconds + 3: Time to load utils op: 0.347059965133667 seconds + 3: Time to load utils op: 0.2170088291168213 seconds + 3: Time to load utils op: 0.3471870422363281 secondsTime to load utils op: 0.34673118591308594 seconds + 3: Time to load utils op: 0.3471856117248535 seconds + 3: + 8: Time to load utils op: 0.21367812156677246 secondsTime to load utils op: 0.21248245239257812 secondsTime to load utils op: 0.2136821746826172 seconds + 8: + 8: + 8: Time to load utils op: 0.340362548828125 secondsTime to load utils op: 0.34036755561828613 seconds + 8: + 8: Time to load utils op: 0.3403749465942383 seconds + 8: Time to load utils op: 0.21326017379760742 secondsTime to load utils op: 0.34037351608276367 seconds + 8: +19: Time to load utils op: 0.20871257781982422 seconds +19: Time to load utils op: 0.20858073234558105 secondsTime to load utils op: 0.20687103271484375 seconds +19: +19: Time to load utils op: 0.33626627922058105 secondsTime to load utils op: 0.20864462852478027 seconds +19: +19: Time to load utils op: 0.3364591598510742 secondsTime to load utils op: 0.33667731285095215 seconds +19: +19: Time to load utils op: 0.33672523498535156 seconds +24: Time to load utils op: 0.20465731620788574 seconds +24: Time to load utils op: 0.33104705810546875 secondsTime to load utils op: 0.20492076873779297 seconds +24: +24: Time to load utils op: 0.33106422424316406 seconds +24: Time to load utils op: 0.33106446266174316 seconds +21: Time to load utils op: 0.20697951316833496 secondsTime to load utils op: 0.20689678192138672 secondsTime to load utils op: 0.2059926986694336 seconds +21: Time to load utils op: 0.1984548568725586 seconds +21: +21: Time to load utils op: 0.334514856338501 seconds +21: Time to load utils op: 0.3345215320587158 seconds +24: Time to load utils op: 0.20495200157165527 secondsTime to load utils op: 0.2048342227935791 seconds +24: +24: Time to load utils op: 0.3311021327972412 seconds +21: +21: Time to load utils op: 0.33452558517456055 seconds +21: Time to load utils op: 0.3345308303833008 seconds + 7: Time to load utils op: 0.21567034721374512 seconds + 7: Time to load utils op: 0.21588683128356934 seconds + 7: Time to load utils op: 0.21568608283996582 secondsTime to load utils op: 0.21588468551635742 seconds + 7: + 7: Time to load utils op: 0.34613513946533203 seconds + 7: Time to load utils op: 0.34601902961730957 secondsTime to load utils op: 0.34611034393310547 secondsTime to load utils op: 0.3461318016052246 seconds + 7: + 7: + 9: Time to load utils op: 0.22143864631652832 seconds + 9: Time to load utils op: 0.34836721420288086 seconds + 9: Time to load utils op: 0.3483750820159912 seconds + 9: Time to load utils op: 0.22111034393310547 seconds + 9: Time to load utils op: 0.2214980125427246 seconds + 9: Time to load utils op: 0.3483898639678955 secondsTime to load utils op: 0.22109317779541016 seconds + 9: + 9: Time to load utils op: 0.34842991828918457 seconds +29: Time to load utils op: 0.0005066394805908203 seconds + 6: Time to load utils op: 0.0004944801330566406 seconds + 6: Time to load utils op: 0.0005199909210205078 seconds + 6: Time to load utils op: 0.0005154609680175781 seconds + 6: Time to load utils op: 0.0005061626434326172 secondsTime to load utils op: 0.0005054473876953125 secondsTime to load utils op: 0.0004889965057373047 secondsTime to load utils op: 0.0005125999450683594 seconds + 6: + 6: + 6: + 6: Time to load utils op: 0.0004570484161376953 seconds + 1: Time to load utils op: 0.0010914802551269531 seconds + 1: Time to load utils op: 0.0014493465423583984 seconds + 1: Time to load utils op: 0.0014243125915527344 seconds + 1: Time to load utils op: 0.0014035701751708984 seconds + 1: Time to load utils op: 0.0014328956604003906 seconds + 1: Time to load utils op: 0.0014896392822265625 secondsTime to load utils op: 0.0014157295227050781 seconds + 1: + 1: Time to load utils op: 0.00150299072265625 seconds +12: Time to load utils op: 0.0005424022674560547 secondsTime to load utils op: 0.0005156993865966797 seconds +12: +12: Time to load utils op: 0.0005431175231933594 seconds +12: Time to load utils op: 0.00044155120849609375 seconds +12: Time to load utils op: 0.0005578994750976562 seconds +12: Time to load utils op: 0.0006327629089355469 secondsTime to load utils op: 0.0006284713745117188 seconds +12: +12: Time to load utils op: 0.0005927085876464844 seconds +29: Time to load utils op: 0.0005686283111572266 seconds +29: Time to load utils op: 0.00045037269592285156 seconds +29: Time to load utils op: 0.00045180320739746094 seconds +29: Time to load utils op: 0.0004820823669433594 seconds +29: Time to load utils op: 0.000438690185546875 seconds +29: Time to load utils op: 0.00046944618225097656 seconds +29: Time to load utils op: 0.0005552768707275391 seconds +27: Time to load utils op: 0.0005626678466796875 seconds +27: Time to load utils op: 0.0004036426544189453 seconds +27: Time to load utils op: 0.0005521774291992188 seconds +27: Time to load utils op: 0.0006208419799804688 secondsTime to load utils op: 0.0006077289581298828 secondsTime to load utils op: 0.0006148815155029297 seconds +27: +27: +27: Time to load utils op: 0.0005424022674560547 secondsTime to load utils op: 0.0005824565887451172 seconds +27: + 3: Time to load utils op: 0.0005028247833251953 seconds + 3: Time to load utils op: 0.0004937648773193359 seconds + 3: Time to load utils op: 0.00041103363037109375 seconds + 3: Time to load utils op: 0.0005447864532470703 seconds + 3: Time to load utils op: 0.00047516822814941406 seconds + 3: Time to load utils op: 0.0004944801330566406 secondsTime to load utils op: 0.0004742145538330078 secondsTime to load utils op: 0.00046539306640625 seconds + 3: + 3: + 0: [2023-05-13 09:38:41,634] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 + 0: [2023-05-13 09:38:41,635] [INFO] [utils.py:828:see_memory_usage] MA 3.99 GB Max_MA 3.99 GB CA 4.05 GB Max_CA 4 GB + 0: [2023-05-13 09:38:41,635] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.59 GB, percent = 8.1% + 5: Time to load utils op: 0.0009198188781738281 seconds + 5: Time to load utils op: 0.0007119178771972656 seconds + 5: Time to load utils op: 0.0007348060607910156 seconds + 5: Time to load utils op: 0.0008862018585205078 secondsTime to load utils op: 0.0008404254913330078 seconds + 5: Time to load utils op: 0.0007984638214111328 seconds + 5: + 5: Time to load utils op: 0.0009391307830810547 secondsTime to load utils op: 0.0008955001831054688 seconds + 5: +21: Time to load utils op: 0.0008509159088134766 seconds +21: Time to load utils op: 0.0008580684661865234 seconds +21: Time to load utils op: 0.0008177757263183594 seconds +21: Time to load utils op: 0.0008327960968017578 secondsTime to load utils op: 0.0008907318115234375 secondsTime to load utils op: 0.0007913112640380859 seconds +21: +21: +21: Time to load utils op: 0.0009341239929199219 seconds +21: Time to load utils op: 0.0008959770202636719 seconds + 2: Time to load utils op: 0.0005035400390625 seconds + 2: Time to load utils op: 0.0004105567932128906 seconds + 2: Time to load utils op: 0.0005390644073486328 seconds + 2: Time to load utils op: 0.00042366981506347656 secondsTime to load utils op: 0.0005512237548828125 seconds + 2: + 2: Time to load utils op: 0.00043892860412597656 seconds + 2: Time to load utils op: 0.0005900859832763672 seconds + 2: Time to load utils op: 0.0004973411560058594 seconds +18: Time to load utils op: 0.0005526542663574219 seconds +18: Time to load utils op: 0.0005385875701904297 secondsTime to load utils op: 0.0005831718444824219 seconds +18: +11: Time to load utils op: 0.0005371570587158203 seconds +11: Time to load utils op: 0.0004391670227050781 secondsTime to load utils op: 0.00043845176696777344 seconds +11: +11: Time to load utils op: 0.0004291534423828125 secondsTime to load utils op: 0.0005476474761962891 seconds +11: +31: Time to load utils op: 0.0005021095275878906 seconds +18: Time to load utils op: 0.0006077289581298828 seconds +18: Time to load utils op: 0.0006051063537597656 seconds +18: Time to load utils op: 0.0006334781646728516 secondsTime to load utils op: 0.0006463527679443359 seconds +11: Time to load utils op: 0.0005807876586914062 seconds +11: Time to load utils op: 0.0005550384521484375 seconds +18: Time to load utils op: 0.0006546974182128906 seconds +18: +31: Time to load utils op: 0.0005242824554443359 seconds +31: Time to load utils op: 0.0004265308380126953 secondsTime to load utils op: 0.00043463706970214844 seconds +31: +11: Time to load utils op: 0.0005619525909423828 seconds +31: Time to load utils op: 0.00041985511779785156 seconds +31: Time to load utils op: 0.0004258155822753906 seconds +31: Time to load utils op: 0.0005915164947509766 seconds +31: Time to load utils op: 0.0006043910980224609 seconds +15: Time to load utils op: 0.0005214214324951172 seconds +10: Time to load utils op: 0.00047898292541503906 seconds +15: Time to load utils op: 0.0005593299865722656 seconds +15: Time to load utils op: 0.0004603862762451172 secondsTime to load utils op: 0.0005407333374023438 seconds +15: +15: Time to load utils op: 0.0006039142608642578 seconds +10: Time to load utils op: 0.0005617141723632812 seconds +10: Time to load utils op: 0.0004334449768066406 seconds +15: Time to load utils op: 0.0006182193756103516 seconds +15: Time to load utils op: 0.0006635189056396484 secondsTime to load utils op: 0.0006749629974365234 seconds +15: +10: Time to load utils op: 0.000568389892578125 seconds +10: Time to load utils op: 0.0005927085876464844 secondsTime to load utils op: 0.00042891502380371094 seconds +10: +10: Time to load utils op: 0.0006039142608642578 seconds +10: Time to load utils op: 0.0006091594696044922 seconds +19: Time to load utils op: 0.0005490779876708984 seconds +19: Time to load utils op: 0.0005471706390380859 seconds +19: Time to load utils op: 0.0006260871887207031 seconds +19: Time to load utils op: 0.0006754398345947266 seconds +19: Time to load utils op: 0.0005698204040527344 secondsTime to load utils op: 0.0006296634674072266 seconds +19: +19: Time to load utils op: 0.0006911754608154297 secondsTime to load utils op: 0.0007081031799316406 seconds +19: +30: Time to load utils op: 0.0005354881286621094 seconds +30: Time to load utils op: 0.0005347728729248047 seconds +30: Time to load utils op: 0.0005564689636230469 seconds +22: Time to load utils op: 0.0004756450653076172 secondsTime to load utils op: 0.0004627704620361328 seconds +22: +30: Time to load utils op: 0.0005590915679931641 secondsTime to load utils op: 0.0005939006805419922 seconds +30: +22: Time to load utils op: 0.00044846534729003906 secondsTime to load utils op: 0.0005524158477783203 seconds +22: +22: Time to load utils op: 0.0005373954772949219 seconds +22: Time to load utils op: 0.0004687309265136719 seconds +30: Time to load utils op: 0.0006003379821777344 seconds +30: Time to load utils op: 0.0006213188171386719 secondsTime to load utils op: 0.0005958080291748047 seconds +30: +22: Time to load utils op: 0.0004410743713378906 seconds +22: Time to load utils op: 0.0005555152893066406 seconds +20: Time to load utils op: 0.0005483627319335938 secondsTime to load utils op: 0.0005662441253662109 seconds +20: +20: Time to load utils op: 0.00042629241943359375 secondsTime to load utils op: 0.0004475116729736328 seconds +20: +25: Time to load utils op: 0.0005407333374023438 seconds +20: Time to load utils op: 0.0004744529724121094 seconds +25: Time to load utils op: 0.0004405975341796875 seconds +20: Time to load utils op: 0.0005958080291748047 secondsTime to load utils op: 0.0005929470062255859 secondsTime to load utils op: 0.0005960464477539062 seconds +20: +20: +26: Time to load utils op: 0.0006167888641357422 secondsTime to load utils op: 0.0005979537963867188 seconds +26: +17: Time to load utils op: 0.0004813671112060547 seconds +17: Time to load utils op: 0.00047135353088378906 secondsTime to load utils op: 0.0004444122314453125 seconds +17: +17: Time to load utils op: 0.0005061626434326172 seconds +17: Time to load utils op: 0.0004923343658447266 secondsTime to load utils op: 0.0005128383636474609 seconds +17: +17: Time to load utils op: 0.0004551410675048828 seconds +25: Time to load utils op: 0.0005908012390136719 secondsTime to load utils op: 0.0005931854248046875 seconds +25: +25: Time to load utils op: 0.0006213188171386719 seconds +25: Time to load utils op: 0.0006265640258789062 seconds +25: Time to load utils op: 0.0006029605865478516 seconds +25: Time to load utils op: 0.0006096363067626953 seconds +17: Time to load utils op: 0.0011162757873535156 seconds + 7: Time to load utils op: 0.0004992485046386719 seconds + 7: Time to load utils op: 0.0003819465637207031 seconds + 7: Time to load utils op: 0.0005044937133789062 seconds + 7: Time to load utils op: 0.0004999637603759766 seconds + 7: Time to load utils op: 0.0004899501800537109 secondsTime to load utils op: 0.0004858970642089844 seconds + 7: + 7: Time to load utils op: 0.000507354736328125 seconds + 7: Time to load utils op: 0.0005269050598144531 seconds +28: Time to load utils op: 0.0007135868072509766 secondsTime to load utils op: 0.0007643699645996094 seconds +28: Time to load utils op: 0.00072479248046875 seconds +28: +28: Time to load utils op: 0.0007443428039550781 seconds +28: Time to load utils op: 0.000812530517578125 secondsTime to load utils op: 0.0010230541229248047 seconds +28: +28: Time to load utils op: 0.0008521080017089844 seconds +28: Time to load utils op: 0.0009534358978271484 seconds +14: Time to load utils op: 0.0017094612121582031 seconds +14: Time to load utils op: 0.0016946792602539062 seconds +14: Time to load utils op: 0.0020935535430908203 seconds +14: Time to load utils op: 0.002028942108154297 secondsTime to load utils op: 0.001935720443725586 seconds +14: +14: Time to load utils op: 0.002047300338745117 seconds +14: Time to load utils op: 0.0020127296447753906 seconds +14: Time to load utils op: 0.0020294189453125 seconds + 4: Time to load utils op: 0.0005335807800292969 seconds + 4: Time to load utils op: 0.0005385875701904297 seconds + 4: Time to load utils op: 0.0008957386016845703 seconds + 4: Time to load utils op: 0.0009717941284179688 seconds + 4: Time to load utils op: 0.0009317398071289062 secondsTime to load utils op: 0.0009264945983886719 secondsTime to load utils op: 0.0008904933929443359 seconds + 4: + 4: + 4: Time to load utils op: 0.000949859619140625 seconds +24: Time to load utils op: 0.0010972023010253906 seconds +24: Time to load utils op: 0.0010106563568115234 seconds +24: Time to load utils op: 0.0011105537414550781 seconds +24: Time to load utils op: 0.0013506412506103516 seconds +24: Time to load utils op: 0.0012593269348144531 secondsTime to load utils op: 0.0013606548309326172 seconds +24: +24: Time to load utils op: 0.001287698745727539 seconds +24: Time to load utils op: 0.0013229846954345703 seconds +13: Time to load utils op: 0.0009198188781738281 seconds +13: Time to load utils op: 0.0009329319000244141 seconds +13: Time to load utils op: 0.0007832050323486328 seconds +13: Time to load utils op: 0.0008561611175537109 secondsTime to load utils op: 0.0007665157318115234 seconds +13: Time to load utils op: 0.0008494853973388672 seconds +13: +13: Time to load utils op: 0.0007903575897216797 seconds +13: Time to load utils op: 0.0008306503295898438 seconds + 9: Time to load utils op: 0.0012176036834716797 seconds + 9: Time to load utils op: 0.001636505126953125 seconds + 9: Time to load utils op: 0.0015990734100341797 seconds + 9: Time to load utils op: 0.0016820430755615234 seconds + 9: Time to load utils op: 0.001680135726928711 seconds + 9: Time to load utils op: 0.0016939640045166016 seconds + 9: Time to load utils op: 0.0016222000122070312 seconds + 9: Time to load utils op: 0.0016512870788574219 seconds + 8: Time to load utils op: 0.0005626678466796875 seconds + 8: Time to load utils op: 0.000568389892578125 seconds + 8: Time to load utils op: 0.0006184577941894531 secondsTime to load utils op: 0.000606536865234375 seconds + 8: + 8: Time to load utils op: 0.0006191730499267578 seconds + 8: Time to load utils op: 0.0006577968597412109 seconds + 8: Time to load utils op: 0.0005996227264404297 secondsTime to load utils op: 0.0006296634674072266 seconds + 8: +16: Time to load utils op: 0.0011854171752929688 seconds +16: Time to load utils op: 0.0014362335205078125 seconds +16: Time to load utils op: 0.0014600753784179688 seconds +16: Time to load utils op: 0.0014386177062988281 seconds +16: Time to load utils op: 0.0014286041259765625 secondsTime to load utils op: 0.0013790130615234375 secondsTime to load utils op: 0.001369476318359375 seconds +16: +16: +16: Time to load utils op: 0.001458883285522461 seconds + 0: Time to load utils op: 0.0005099773406982422 seconds + 0: Time to load utils op: 0.0005238056182861328 seconds + 0: Time to load utils op: 0.0004723072052001953 secondsTime to load utils op: 0.0005016326904296875 secondsTime to load utils op: 0.00046133995056152344 seconds + 0: Time to load utils op: 0.00047588348388671875 seconds + 0: + 0: + 0: Time to load utils op: 0.0005030632019042969 seconds +26: Time to load utils op: 0.0010762214660644531 seconds +26: Time to load utils op: 0.001094818115234375 seconds +26: Time to load utils op: 0.0010602474212646484 seconds +26: Time to load utils op: 0.0011012554168701172 seconds +26: Time to load utils op: 0.001062154769897461 seconds +26: Time to load utils op: 0.0010645389556884766 seconds + 0: [2023-05-13 09:38:41,790] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 + 0: [2023-05-13 09:38:41,791] [INFO] [utils.py:828:see_memory_usage] MA 8.04 GB Max_MA 8.04 GB CA 10.14 GB Max_CA 10 GB + 0: [2023-05-13 09:38:41,791] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.7 GB, percent = 8.1% + 0: [2023-05-13 09:38:41,903] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 + 0: [2023-05-13 09:38:41,903] [INFO] [utils.py:828:see_memory_usage] MA 8.04 GB Max_MA 8.04 GB CA 10.14 GB Max_CA 10 GB + 0: [2023-05-13 09:38:41,903] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.7 GB, percent = 8.1% + 0: [2023-05-13 09:38:42,007] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 + 0: [2023-05-13 09:38:42,008] [INFO] [utils.py:828:see_memory_usage] MA 11.92 GB Max_MA 11.92 GB CA 15.94 GB Max_CA 16 GB + 0: [2023-05-13 09:38:42,008] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.7 GB, percent = 8.1% +23: Time to load utils op: 0.0008099079132080078 seconds +23: Time to load utils op: 0.0009322166442871094 seconds +23: Time to load utils op: 0.0008680820465087891 seconds +23: Time to load utils op: 0.0008184909820556641 seconds +23: Time to load utils op: 0.00083160400390625 secondsTime to load utils op: 0.0008101463317871094 seconds +23: +23: Time to load utils op: 0.0008473396301269531 seconds +23: Time to load utils op: 0.0008747577667236328 seconds + 0: [2023-05-13 09:38:42,108] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 + 0: [2023-05-13 09:38:42,108] [INFO] [utils.py:828:see_memory_usage] MA 11.92 GB Max_MA 11.92 GB CA 15.94 GB Max_CA 16 GB + 0: [2023-05-13 09:38:42,109] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.7 GB, percent = 8.1% + 0: [2023-05-13 09:38:42,215] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 + 0: [2023-05-13 09:38:42,215] [INFO] [utils.py:828:see_memory_usage] MA 11.93 GB Max_MA 11.93 GB CA 15.94 GB Max_CA 16 GB + 0: [2023-05-13 09:38:42,215] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.7 GB, percent = 8.1% + 0: [2023-05-13 09:38:42,319] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer + 0: [2023-05-13 09:38:42,320] [INFO] [utils.py:828:see_memory_usage] MA 11.93 GB Max_MA 11.93 GB CA 15.94 GB Max_CA 16 GB + 0: [2023-05-13 09:38:42,320] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.7 GB, percent = 8.1% + 0: [2023-05-13 09:38:42,431] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer + 0: [2023-05-13 09:38:42,431] [INFO] [utils.py:828:see_memory_usage] MA 12.05 GB Max_MA 12.05 GB CA 15.94 GB Max_CA 16 GB + 0: [2023-05-13 09:38:42,431] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.7 GB, percent = 8.1% + 0: [2023-05-13 09:38:42,535] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer + 0: [2023-05-13 09:38:42,536] [INFO] [utils.py:828:see_memory_usage] MA 12.05 GB Max_MA 12.05 GB CA 15.94 GB Max_CA 16 GB + 0: [2023-05-13 09:38:42,536] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.7 GB, percent = 8.1% + 0: [2023-05-13 09:38:42,536] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam + 0: [2023-05-13 09:38:42,536] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler + 0: [2023-05-13 09:38:42,536] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = + 0: [2023-05-13 09:38:42,536] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] + 0: [2023-05-13 09:38:42,537] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: + 0: [2023-05-13 09:38:42,537] [INFO] [config.py:1011:print] activation_checkpointing_config { + 0: "partition_activations": false, + 0: "contiguous_memory_optimization": false, + 0: "cpu_checkpointing": false, + 0: "number_checkpoints": null, + 0: "synchronize_checkpoint_boundary": false, + 0: "profile": false + 0: } + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] amp_enabled .................. False + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] amp_params ................... False + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] autotuning_config ............ { + 0: "enabled": false, + 0: "start_step": null, + 0: "end_step": null, + 0: "metric_path": null, + 0: "arg_mappings": null, + 0: "metric": "throughput", + 0: "model_info": null, + 0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", + 0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", + 0: "overwrite": true, + 0: "fast": true, + 0: "start_profile_step": 3, + 0: "end_profile_step": 5, + 0: "tuner_type": "gridsearch", + 0: "tuner_early_stopping": 5, + 0: "tuner_num_trials": 50, + 0: "model_info_path": null, + 0: "mp_size": 1, + 0: "max_train_batch_size": null, + 0: "min_train_batch_size": 1, + 0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, + 0: "min_train_micro_batch_size_per_gpu": 1, + 0: "num_tuning_micro_batch_sizes": 3 + 0: } + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] bfloat16_enabled ............. True + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] comms_config ................. + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] communication_data_type ...... None + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa + 0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] curriculum_enabled ........... False + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] curriculum_params ............ False + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] dataloader_drop_last ......... False + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] disable_allgather ............ False + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] dump_state ................... False + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] elasticity_enabled ........... False + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] flops_profiler_config ........ { + 0: "enabled": false, + 0: "profile_step": 1, + 0: "module_depth": -1, + 0: "top_modules": 1, + 0: "detailed": true, + 0: "output_file": null + 0: } + 0: [2023-05-13 09:38:42,538] [INFO] [config.py:1011:print] fp16_auto_cast ............... None + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] fp16_enabled ................. False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] global_rank .................. 0 + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 4 + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] load_universal_checkpoint .... False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] loss_scale ................... 1.0 + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] memory_breakdown ............. False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] monitor_config ............... + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] nebula_config ................ { + 0: "enabled": false, + 0: "persistent_storage_path": null, + 0: "persistent_time_interval": 100, + 0: "num_of_version_in_retention": 2, + 0: "enable_nebula_load": true, + 0: "load_path": null + 0: } + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] optimizer_name ............... None + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] optimizer_params ............. None + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] pld_enabled .................. False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] pld_params ................... False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] prescale_gradients ........... False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] scheduler_name ............... None + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] scheduler_params ............. None + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] sparse_attention ............. None + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] steps_per_print .............. 2000 + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] train_batch_size ............. 1024 + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 2 + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] use_node_local_storage ....... False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] world_size ................... 128 + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] zero_enabled ................. False + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 + 0: [2023-05-13 09:38:42,539] [INFO] [config.py:996:print_user_config] json = { + 0: "train_micro_batch_size_per_gpu": 2, + 0: "train_batch_size": 1.024000e+03, + 0: "gradient_clipping": 1.0, + 0: "zero_optimization": { + 0: "stage": 0 + 0: }, + 0: "bf16": { + 0: "enabled": true + 0: }, + 0: "steps_per_print": 2.000000e+03, + 0: "wall_clock_breakdown": false + 0: } + 0: Time to load utils op: 0.0004324913024902344 seconds + 0: [2023-05-13 09:38:42,540] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=4 micro_batch_size=2 + 0: [2023-05-13 09:38:42,563] [INFO] [engine.py:145:__init__] RANK=1 STAGE=0 LAYERS=43 [0, 43) STAGE_PARAMS=2123243520 (2123.244M) TOTAL_PARAMS=4246487040 (4246.487M) UNIQUE_PARAMS=4246487040 (4246.487M) + 0: [2023-05-13 09:38:42,563] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=43 [0, 43) STAGE_PARAMS=2123243520 (2123.244M) TOTAL_PARAMS=4246487040 (4246.487M) UNIQUE_PARAMS=4246487040 (4246.487M) +30: [2023-05-13 09:38:44,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:44,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:44,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:44,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:44,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:44,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:44,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:44,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:44,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:44,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:44,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:44,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:44,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:44,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:44,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:44,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:44,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:44,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:44,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:44,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:44,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:44,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:44,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:44,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:44,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +27: [2023-05-13 09:38:44,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 7: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 7: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 7: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 7: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +27: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +27: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +27: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +19: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +27: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +19: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +22: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +27: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:44,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +19: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +19: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +19: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +24: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 6: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +11: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:44,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 7: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +11: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +15: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +19: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +24: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +11: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +11: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +15: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +24: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +19: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +22: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +22: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 2: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +16: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +22: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:44,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +19: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 6: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:44,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 6: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +16: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +11: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 2: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +11: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +15: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +25: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +25: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +25: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +13: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 4: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +12: [2023-05-13 09:38:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +28: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:44,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 4: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +13: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +25: [2023-05-13 09:38:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 7: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 7: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 7: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +27: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +15: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +27: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +27: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 7: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +24: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +19: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +11: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +11: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +11: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +24: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +15: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +13: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 6: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +15: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +27: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 2: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +13: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +13: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +24: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +15: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +11: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +24: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 7: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +24: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +25: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 4: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +24: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +11: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +22: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +16: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +12: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +28: [2023-05-13 09:38:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +16: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +12: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +12: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +12: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +10: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +20: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +17: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 1: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +20: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +20: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 0: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +26: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +20: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +20: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +20: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +10: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +10: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +26: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +10: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +26: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +26: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +26: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 9: [2023-05-13 09:38:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +21: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +17: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +20: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 9: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +10: [2023-05-13 09:38:44,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +26: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +21: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +20: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +21: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +20: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +20: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +20: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +20: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:44,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 0: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 0: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +26: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +20: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +26: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +10: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +10: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +26: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 0: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +17: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 0: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 0: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +26: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +26: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 1: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 0: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +26: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +10: [2023-05-13 09:38:44,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +26: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 1: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 0: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:44,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:44,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:44,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +30: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +30: [2023-05-13 09:38:44,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +30: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +30: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +29: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 3: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +18: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +23: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:44,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +31: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 8: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +31: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 3: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +31: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 8: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +18: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +23: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +18: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +23: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 3: [2023-05-13 09:38:44,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +23: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +14: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +23: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... +18: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +14: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +14: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... +14: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 3: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:44,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 8: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 8: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +29: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +23: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +23: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +29: [2023-05-13 09:38:44,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 8: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 8: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +23: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:44,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +14: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. +29: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:44,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:44,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:44,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:44,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +29: [2023-05-13 09:38:44,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:44,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:44,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:44,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:44,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +31: [2023-05-13 09:38:44,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +14: [2023-05-13 09:38:44,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +14: [2023-05-13 09:38:44,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +14: [2023-05-13 09:38:44,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. +31: [2023-05-13 09:38:44,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:44,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:44,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:44,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:44,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:44,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:44,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:44,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt... + 5: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt... + 5: [2023-05-13 09:38:44,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:44,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:44,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:44,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_00_model_states.pt. + 5: [2023-05-13 09:38:44,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:44,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:44,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:44,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/mp_rank_01_model_states.pt. + 5: [2023-05-13 09:38:44,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:44,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:45,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:45,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:45,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:45,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:45,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:45,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:45,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:45,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:45,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:45,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:45,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:45,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:45,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +27: [2023-05-13 09:38:45,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:45,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:45,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:45,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:45,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:45,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:45,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:45,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:45,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:45,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:45,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:45,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:45,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:45,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:45,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +22: [2023-05-13 09:38:45,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:45,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:45,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:45,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:45,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:45,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:45,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:45,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:45,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:45,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:45,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:45,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:45,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:45,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +13: [2023-05-13 09:38:45,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:45,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:45,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +21: [2023-05-13 09:38:45,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +21: [2023-05-13 09:38:45,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:45,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:45,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:45,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:45,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:45,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:45,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:45,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:45,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:45,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:45,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:45,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:45,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:45,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:45,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:45,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:45,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:45,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:45,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:45,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:45,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:45,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +13: [2023-05-13 09:38:45,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +18: [2023-05-13 09:38:45,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:45,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +18: [2023-05-13 09:38:45,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:45,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:45,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:45,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:45,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:45,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:45,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +16: [2023-05-13 09:38:45,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +16: [2023-05-13 09:38:45,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +16: [2023-05-13 09:38:45,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:45,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +16: [2023-05-13 09:38:45,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:45,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +25: [2023-05-13 09:38:45,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:45,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:45,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:45,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:45,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +18: [2023-05-13 09:38:45,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +18: [2023-05-13 09:38:45,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +18: [2023-05-13 09:38:45,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +18: [2023-05-13 09:38:45,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +18: [2023-05-13 09:38:45,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:45,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +18: [2023-05-13 09:38:45,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:45,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:45,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:45,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:45,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:45,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:45,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:45,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:45,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:45,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +11: [2023-05-13 09:38:45,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:45,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:45,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +27: [2023-05-13 09:38:45,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:45,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:45,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:45,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:45,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:45,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:45,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:45,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:45,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +28: [2023-05-13 09:38:45,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:45,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:45,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +23: [2023-05-13 09:38:45,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:45,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +15: [2023-05-13 09:38:45,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:45,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +23: [2023-05-13 09:38:45,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:45,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:45,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:45,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:45,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +15: [2023-05-13 09:38:45,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:45,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:45,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:45,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:45,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:45,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:45,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:45,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:45,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:45,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +27: [2023-05-13 09:38:45,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:45,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:45,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:45,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +13: [2023-05-13 09:38:45,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:45,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +13: [2023-05-13 09:38:45,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:45,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +15: [2023-05-13 09:38:45,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:45,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:45,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:45,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:45,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +22: [2023-05-13 09:38:45,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:45,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:45,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:45,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:45,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:45,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +25: [2023-05-13 09:38:45,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:45,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:45,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:45,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:45,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +21: [2023-05-13 09:38:45,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 5: [2023-05-13 09:38:45,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +22: [2023-05-13 09:38:45,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:45,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:45,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:45,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +13: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +16: [2023-05-13 09:38:45,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:45,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:45,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:45,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +18: [2023-05-13 09:38:45,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:45,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:45,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +19: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +26: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +30: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +19: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +19: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +25: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +19: [2023-05-13 09:38:45,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +11: [2023-05-13 09:38:45,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:45,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:45,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +19: [2023-05-13 09:38:45,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +16: [2023-05-13 09:38:45,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:45,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:45,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +17: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +26: [2023-05-13 09:38:45,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:45,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:45,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:45,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:45,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +14: [2023-05-13 09:38:45,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +31: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +10: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:45,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:45,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:45,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:45,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:45,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:45,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:45,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +30: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +10: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +11: [2023-05-13 09:38:45,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:45,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +28: [2023-05-13 09:38:45,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:45,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +31: [2023-05-13 09:38:45,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:45,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:45,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +28: [2023-05-13 09:38:45,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:45,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +17: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:45,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:45,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +17: [2023-05-13 09:38:45,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +23: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +10: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +23: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +17: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:45,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:45,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +17: [2023-05-13 09:38:45,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:45,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:45,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +10: [2023-05-13 09:38:45,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:45,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:45,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +12: [2023-05-13 09:38:45,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +10: [2023-05-13 09:38:45,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:45,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:45,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:45,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:45,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:45,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:45,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:45,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +12: [2023-05-13 09:38:45,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:45,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:45,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:45,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:45,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +29: [2023-05-13 09:38:45,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:45,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +29: [2023-05-13 09:38:45,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +23: [2023-05-13 09:38:45,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:45,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +14: [2023-05-13 09:38:45,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +20: [2023-05-13 09:38:45,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +20: [2023-05-13 09:38:45,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:45,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:45,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +14: [2023-05-13 09:38:45,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:45,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:45,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... +24: [2023-05-13 09:38:45,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt... +24: [2023-05-13 09:38:45,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +23: [2023-05-13 09:38:45,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +30: [2023-05-13 09:38:45,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:45,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:45,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:45,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:45,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:45,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:45,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:45,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:45,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +10: [2023-05-13 09:38:45,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:45,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +29: [2023-05-13 09:38:45,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:45,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +19: [2023-05-13 09:38:45,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:45,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:45,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:45,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +26: [2023-05-13 09:38:45,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:45,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:45,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:45,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:45,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:45,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:45,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:45,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:45,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +24: [2023-05-13 09:38:45,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +31: [2023-05-13 09:38:45,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +20: [2023-05-13 09:38:45,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +17: [2023-05-13 09:38:45,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_00-model_states.pt. +12: [2023-05-13 09:38:45,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:45,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:45,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:45,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:45,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:45,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:45,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:45,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:45,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:45,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:45,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +13: [2023-05-13 09:38:45,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +13: [2023-05-13 09:38:45,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +13: [2023-05-13 09:38:45,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +13: [2023-05-13 09:38:45,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:45,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:45,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:45,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:45,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +22: [2023-05-13 09:38:45,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +22: [2023-05-13 09:38:45,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +22: [2023-05-13 09:38:45,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +22: [2023-05-13 09:38:45,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:45,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:45,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:45,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +11: [2023-05-13 09:38:45,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 3: [2023-05-13 09:38:45,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 3: [2023-05-13 09:38:45,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 3: [2023-05-13 09:38:45,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +11: [2023-05-13 09:38:45,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:45,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:45,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +23: [2023-05-13 09:38:45,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +23: [2023-05-13 09:38:45,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +23: [2023-05-13 09:38:45,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:45,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +28: [2023-05-13 09:38:45,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +28: [2023-05-13 09:38:45,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +28: [2023-05-13 09:38:45,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 6: [2023-05-13 09:38:45,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:45,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:45,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:45,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:45,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +11: [2023-05-13 09:38:45,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +11: [2023-05-13 09:38:45,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +11: [2023-05-13 09:38:45,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +11: [2023-05-13 09:38:45,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:45,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:45,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:45,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:45,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:45,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:45,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +27: [2023-05-13 09:38:45,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +27: [2023-05-13 09:38:45,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +27: [2023-05-13 09:38:45,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 9: [2023-05-13 09:38:45,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:45,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:45,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +16: [2023-05-13 09:38:45,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +16: [2023-05-13 09:38:45,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +26: [2023-05-13 09:38:45,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +26: [2023-05-13 09:38:45,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +26: [2023-05-13 09:38:45,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +26: [2023-05-13 09:38:45,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +14: [2023-05-13 09:38:45,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +14: [2023-05-13 09:38:45,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 8: [2023-05-13 09:38:45,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +16: [2023-05-13 09:38:45,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:45,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:45,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:45,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +18: [2023-05-13 09:38:45,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +18: [2023-05-13 09:38:45,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +18: [2023-05-13 09:38:45,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:45,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:45,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:45,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:45,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:45,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:45,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +15: [2023-05-13 09:38:45,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:45,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:45,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:45,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +15: [2023-05-13 09:38:45,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:45,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:45,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:45,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:45,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:45,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:45,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:45,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:45,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:45,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:45,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:45,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:45,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:45,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:45,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:45,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:45,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:45,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:45,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +16: [2023-05-13 09:38:45,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:45,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:45,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:45,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:45,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:45,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +29: [2023-05-13 09:38:45,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:45,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +18: [2023-05-13 09:38:45,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +29: [2023-05-13 09:38:45,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:45,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:45,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 7: [2023-05-13 09:38:45,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +30: [2023-05-13 09:38:45,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 7: [2023-05-13 09:38:45,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +29: [2023-05-13 09:38:45,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:45,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:45,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:45,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +25: [2023-05-13 09:38:45,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:45,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +26: [2023-05-13 09:38:45,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:45,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:45,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:45,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +26: [2023-05-13 09:38:45,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:45,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 9: [2023-05-13 09:38:45,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 9: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +25: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 5: [2023-05-13 09:38:45,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:45,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 9: [2023-05-13 09:38:45,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +29: [2023-05-13 09:38:45,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +27: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +27: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:45,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:45,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:45,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:45,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:45,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:45,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +27: [2023-05-13 09:38:45,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +11: [2023-05-13 09:38:45,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +11: [2023-05-13 09:38:45,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +23: [2023-05-13 09:38:45,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:45,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:45,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:45,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:45,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:45,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:45,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:45,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +10: [2023-05-13 09:38:45,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 4: [2023-05-13 09:38:45,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:45,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:45,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:45,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:45,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:45,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:45,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 3: [2023-05-13 09:38:45,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:45,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:45,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +10: [2023-05-13 09:38:45,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +11: [2023-05-13 09:38:45,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +11: [2023-05-13 09:38:45,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:45,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:45,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:45,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:45,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +23: [2023-05-13 09:38:45,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +14: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +15: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 8: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +11: [2023-05-13 09:38:45,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +11: [2023-05-13 09:38:45,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +26: [2023-05-13 09:38:45,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:45,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:45,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:45,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:45,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 3: [2023-05-13 09:38:45,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:45,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:45,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:45,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +14: [2023-05-13 09:38:45,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 3: [2023-05-13 09:38:45,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:45,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:45,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +26: [2023-05-13 09:38:45,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:45,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:45,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:45,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 1: [2023-05-13 09:38:45,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 1: [2023-05-13 09:38:45,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:45,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:45,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:45,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:45,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:45,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +23: [2023-05-13 09:38:45,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:45,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:45,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +23: [2023-05-13 09:38:45,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +23: [2023-05-13 09:38:45,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +14: [2023-05-13 09:38:45,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +15: [2023-05-13 09:38:45,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:45,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:45,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:45,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +12: [2023-05-13 09:38:45,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +12: [2023-05-13 09:38:45,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +12: [2023-05-13 09:38:45,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:45,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:45,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +22: [2023-05-13 09:38:45,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:45,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:45,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:45,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:45,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 0: [2023-05-13 09:38:45,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +22: [2023-05-13 09:38:45,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:45,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +25: [2023-05-13 09:38:45,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:45,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:45,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +17: [2023-05-13 09:38:45,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +14: [2023-05-13 09:38:45,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:45,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 8: [2023-05-13 09:38:45,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 7: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:45,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:45,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:45,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:45,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:45,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:45,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:45,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:45,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +23: [2023-05-13 09:38:45,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:45,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:45,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:45,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 4: [2023-05-13 09:38:45,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +25: [2023-05-13 09:38:45,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 2: [2023-05-13 09:38:45,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. + 1: [2023-05-13 09:38:45,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +27: [2023-05-13 09:38:45,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:45,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +25: [2023-05-13 09:38:45,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:45,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:45,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:45,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:45,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 4: [2023-05-13 09:38:45,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:45,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:45,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:45,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:45,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +26: [2023-05-13 09:38:45,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:45,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +27: [2023-05-13 09:38:45,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +22: [2023-05-13 09:38:45,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:45,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:45,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:45,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:45,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:45,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:45,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +26: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +22: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +21: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +13: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:45,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +18: [2023-05-13 09:38:45,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +28: [2023-05-13 09:38:45,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:45,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +13: [2023-05-13 09:38:45,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:45,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:45,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +29: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +18: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +29: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +31: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +31: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +21: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +18: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +18: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +18: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +18: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +18: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +24: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +24: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +31: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +22: [2023-05-13 09:38:45,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +16: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +16: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +24: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 6: [2023-05-13 09:38:45,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +30: [2023-05-13 09:38:45,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +30: [2023-05-13 09:38:45,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:45,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +30: [2023-05-13 09:38:45,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +24: [2023-05-13 09:38:45,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:45,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +24: [2023-05-13 09:38:45,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +28: [2023-05-13 09:38:45,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +27: [2023-05-13 09:38:45,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:45,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:45,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:45,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:45,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:45,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +28: [2023-05-13 09:38:45,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:45,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:45,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:45,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:45,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:45,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:45,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:45,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +13: [2023-05-13 09:38:45,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:45,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:45,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:45,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:45,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:45,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +27: [2023-05-13 09:38:45,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:45,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:45,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:45,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:45,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +31: [2023-05-13 09:38:45,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:45,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +20: [2023-05-13 09:38:45,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +20: [2023-05-13 09:38:45,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +20: [2023-05-13 09:38:45,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +17: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +10: [2023-05-13 09:38:45,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +10: [2023-05-13 09:38:45,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +12: [2023-05-13 09:38:45,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +24: [2023-05-13 09:38:45,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:45,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +21: [2023-05-13 09:38:45,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +10: [2023-05-13 09:38:45,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +21: [2023-05-13 09:38:45,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:45,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:45,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:45,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +19: [2023-05-13 09:38:45,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_01-model_01-model_states.pt. +30: [2023-05-13 09:38:45,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:45,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +21: [2023-05-13 09:38:45,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +18: [2023-05-13 09:38:45,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +12: [2023-05-13 09:38:45,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:45,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +16: [2023-05-13 09:38:45,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 5: [2023-05-13 09:38:45,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:45,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:45,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:45,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 0: [2023-05-13 09:38:45,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:45,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +20: [2023-05-13 09:38:45,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:45,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:45,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +12: [2023-05-13 09:38:45,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:45,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 0: [2023-05-13 09:38:45,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:45,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:45,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +19: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +17: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +16: [2023-05-13 09:38:45,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:45,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:45,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:45,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:45,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:45,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +21: [2023-05-13 09:38:45,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:45,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:45,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt... +30: [2023-05-13 09:38:45,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:45,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +30: [2023-05-13 09:38:45,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:45,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:45,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:45,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +12: [2023-05-13 09:38:45,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 5: [2023-05-13 09:38:45,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:45,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:45,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:45,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:45,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:45,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:45,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:45,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:45,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:45,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +17: [2023-05-13 09:38:45,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:45,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:45,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +31: [2023-05-13 09:38:45,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:45,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 2: [2023-05-13 09:38:45,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:45,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +17: [2023-05-13 09:38:45,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +17: [2023-05-13 09:38:45,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:45,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:45,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:45,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +31: [2023-05-13 09:38:45,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. + 6: [2023-05-13 09:38:45,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:45,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:45,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:45,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:45,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +28: [2023-05-13 09:38:45,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:45,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:45,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:45,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:45,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 2: [2023-05-13 09:38:45,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:45,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +24: [2023-05-13 09:38:45,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:45,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:45,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:45,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +24: [2023-05-13 09:38:45,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +24: [2023-05-13 09:38:45,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +24: [2023-05-13 09:38:45,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +24: [2023-05-13 09:38:45,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:46,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:46,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:46,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:46,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +19: [2023-05-13 09:38:46,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:46,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:46,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +19: [2023-05-13 09:38:46,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:46,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:46,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:46,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:46,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_00-model_states.pt. +20: [2023-05-13 09:38:46,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:46,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... +20: [2023-05-13 09:38:46,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt... + 8: [2023-05-13 09:38:46,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:46,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:46,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:46,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:46,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:46,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:46,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:46,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:46,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:46,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:46,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:46,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +11: [2023-05-13 09:38:46,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +11: [2023-05-13 09:38:46,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 8: [2023-05-13 09:38:46,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:46,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:46,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:46,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:46,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:46,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:46,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:46,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:46,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:46,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:46,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:46,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:46,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:46,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:46,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:46,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +11: [2023-05-13 09:38:46,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:46,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +26: [2023-05-13 09:38:46,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:46,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:46,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:46,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:46,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:46,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:46,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:46,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:46,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:46,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:46,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:46,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:46,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:46,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:46,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:46,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:46,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:46,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:46,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:46,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:46,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:46,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:46,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:46,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:46,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:46,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:46,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:46,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:46,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:46,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:46,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:46,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:46,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:46,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:46,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:46,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:46,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:46,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +25: [2023-05-13 09:38:46,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +25: [2023-05-13 09:38:46,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:46,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:46,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +11: [2023-05-13 09:38:46,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:46,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:46,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:46,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +28: [2023-05-13 09:38:46,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:46,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:46,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:46,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:46,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:46,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:46,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:46,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:46,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:46,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:46,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:46,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:46,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:46,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:46,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:46,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:46,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:46,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:46,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:46,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +15: [2023-05-13 09:38:46,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +15: [2023-05-13 09:38:46,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +29: [2023-05-13 09:38:46,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +29: [2023-05-13 09:38:46,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +15: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +15: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 7: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +15: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:46,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:46,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:46,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:46,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:46,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:46,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:46,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:46,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:46,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +25: [2023-05-13 09:38:46,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:46,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:46,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:46,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:46,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:46,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:46,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +11: [2023-05-13 09:38:46,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:46,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:46,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:46,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +21: [2023-05-13 09:38:46,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +21: [2023-05-13 09:38:46,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +21: [2023-05-13 09:38:46,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +11: [2023-05-13 09:38:46,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:46,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:46,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:46,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:46,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:46,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:46,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:46,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:46,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:46,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:46,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:46,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:46,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:46,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:46,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:46,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:46,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +16: [2023-05-13 09:38:46,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +16: [2023-05-13 09:38:46,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:46,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +12: [2023-05-13 09:38:46,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +12: [2023-05-13 09:38:46,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +12: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 1: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +30: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:46,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +18: [2023-05-13 09:38:46,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:46,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:46,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:46,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:46,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:46,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:46,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +29: [2023-05-13 09:38:46,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +13: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +13: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +13: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +13: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +23: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +24: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +24: [2023-05-13 09:38:46,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +24: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +20: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +20: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +20: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +22: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +10: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 6: [2023-05-13 09:38:46,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:46,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:46,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:46,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +27: [2023-05-13 09:38:46,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:46,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:46,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:46,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:46,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:46,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:46,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:46,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:46,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:46,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:46,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:46,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:46,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +11: [2023-05-13 09:38:46,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:46,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +15: [2023-05-13 09:38:46,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 1: [2023-05-13 09:38:46,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 1: [2023-05-13 09:38:46,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 1: [2023-05-13 09:38:46,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +16: [2023-05-13 09:38:46,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:46,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:46,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:46,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:46,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:46,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +27: [2023-05-13 09:38:46,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +27: [2023-05-13 09:38:46,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +11: [2023-05-13 09:38:46,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:46,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:46,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:46,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:46,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:46,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:46,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:46,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:46,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:46,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:46,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:46,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:46,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:46,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:46,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:46,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:46,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +11: [2023-05-13 09:38:46,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:46,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:46,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:46,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:46,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:46,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:46,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:46,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:46,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:46,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:46,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:46,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:46,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:46,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:46,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:46,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:46,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:46,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:46,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:46,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:46,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:46,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 3: [2023-05-13 09:38:46,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 9: [2023-05-13 09:38:46,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +15: [2023-05-13 09:38:46,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:46,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:46,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:46,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:46,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 3: [2023-05-13 09:38:46,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:46,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:46,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:46,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:46,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:46,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:46,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:46,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:46,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:46,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:46,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:46,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:46,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:46,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:46,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:46,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:46,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:46,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:46,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:46,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:46,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:46,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:46,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 9: [2023-05-13 09:38:46,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:46,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +29: [2023-05-13 09:38:46,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +29: [2023-05-13 09:38:46,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:46,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 9: [2023-05-13 09:38:46,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +18: [2023-05-13 09:38:46,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:46,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:46,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:46,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:46,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 9: [2023-05-13 09:38:46,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +29: [2023-05-13 09:38:46,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +29: [2023-05-13 09:38:46,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +29: [2023-05-13 09:38:46,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +29: [2023-05-13 09:38:46,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +27: [2023-05-13 09:38:46,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:46,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +18: [2023-05-13 09:38:46,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +18: [2023-05-13 09:38:46,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +27: [2023-05-13 09:38:46,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:46,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:46,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:46,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:46,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:46,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:46,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:46,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:46,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 4: [2023-05-13 09:38:46,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 4: [2023-05-13 09:38:46,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 4: [2023-05-13 09:38:46,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 4: [2023-05-13 09:38:46,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 0: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +31: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +19: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +14: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:46,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:46,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 5: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 5: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +19: [2023-05-13 09:38:46,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +31: [2023-05-13 09:38:46,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 0: [2023-05-13 09:38:46,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +31: [2023-05-13 09:38:46,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 7: [2023-05-13 09:38:46,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 0: [2023-05-13 09:38:46,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:46,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 7: [2023-05-13 09:38:46,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 2: [2023-05-13 09:38:46,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. + 2: [2023-05-13 09:38:46,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:46,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +14: [2023-05-13 09:38:46,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:46,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:46,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:46,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:46,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +17: [2023-05-13 09:38:46,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_03-model_01-model_states.pt. +14: [2023-05-13 09:38:46,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +14: [2023-05-13 09:38:46,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:46,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:46,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:46,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:46,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:46,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:46,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +26: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +26: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +23: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 3: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +31: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +26: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +21: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +21: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +21: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +21: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +26: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +23: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +23: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:46,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:46,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +23: [2023-05-13 09:38:46,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +18: [2023-05-13 09:38:46,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:46,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:46,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:46,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:46,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:46,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +22: [2023-05-13 09:38:46,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +22: [2023-05-13 09:38:46,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 6: [2023-05-13 09:38:46,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:46,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:46,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:46,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +22: [2023-05-13 09:38:46,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 6: [2023-05-13 09:38:46,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:46,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:46,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:46,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:46,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:46,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:46,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:46,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:46,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:46,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:46,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:46,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:46,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 4: [2023-05-13 09:38:46,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:46,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 1: [2023-05-13 09:38:46,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 4: [2023-05-13 09:38:46,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:46,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 0: [2023-05-13 09:38:46,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 0: [2023-05-13 09:38:46,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 0: [2023-05-13 09:38:46,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:46,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:46,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:46,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:46,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:46,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:46,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:46,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:46,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:46,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:46,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:46,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:46,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 1: [2023-05-13 09:38:46,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:46,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:46,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:46,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 8: [2023-05-13 09:38:46,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 0: [2023-05-13 09:38:46,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:46,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:46,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:46,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:46,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:46,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:46,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:46,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:46,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +31: [2023-05-13 09:38:46,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +27: [2023-05-13 09:38:46,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 0: [2023-05-13 09:38:46,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +17: [2023-05-13 09:38:46,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +18: [2023-05-13 09:38:46,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:46,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +24: [2023-05-13 09:38:46,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:46,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +18: [2023-05-13 09:38:46,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:46,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +18: [2023-05-13 09:38:46,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:46,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:46,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:46,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:46,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:46,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:46,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +24: [2023-05-13 09:38:46,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:46,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:46,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +27: [2023-05-13 09:38:46,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:46,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 7: [2023-05-13 09:38:46,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:46,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 7: [2023-05-13 09:38:46,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:46,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:46,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:46,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:46,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +21: [2023-05-13 09:38:46,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:46,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:46,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:46,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:46,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:46,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +25: [2023-05-13 09:38:46,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:46,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +26: [2023-05-13 09:38:46,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:46,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +25: [2023-05-13 09:38:46,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 6: [2023-05-13 09:38:46,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +24: [2023-05-13 09:38:46,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:46,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:46,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:46,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:46,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:46,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 3: [2023-05-13 09:38:46,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 7: [2023-05-13 09:38:46,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:46,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:46,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:46,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:46,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:46,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:46,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:46,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +21: [2023-05-13 09:38:46,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:46,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:46,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:46,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:46,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:46,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:46,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:46,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +26: [2023-05-13 09:38:46,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:46,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:46,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:46,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:46,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:46,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:46,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 4: [2023-05-13 09:38:46,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:46,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:46,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:46,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 4: [2023-05-13 09:38:46,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +31: [2023-05-13 09:38:46,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 3: [2023-05-13 09:38:46,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:46,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:46,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 4: [2023-05-13 09:38:46,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 4: [2023-05-13 09:38:46,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:46,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 5: [2023-05-13 09:38:46,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:46,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:46,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 2: [2023-05-13 09:38:46,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:46,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:46,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:46,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 5: [2023-05-13 09:38:46,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +31: [2023-05-13 09:38:46,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +31: [2023-05-13 09:38:46,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:46,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:46,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 8: [2023-05-13 09:38:46,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:46,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:46,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... + 2: [2023-05-13 09:38:46,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:46,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 8: [2023-05-13 09:38:46,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:46,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:46,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +25: [2023-05-13 09:38:46,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +31: [2023-05-13 09:38:46,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +25: [2023-05-13 09:38:46,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:46,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:46,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:46,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:46,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:46,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:46,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:46,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +12: [2023-05-13 09:38:46,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +12: [2023-05-13 09:38:46,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +12: [2023-05-13 09:38:46,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +12: [2023-05-13 09:38:46,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:46,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +12: [2023-05-13 09:38:46,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:46,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +19: [2023-05-13 09:38:46,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +19: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:46,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:46,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:46,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:46,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:46,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +30: [2023-05-13 09:38:46,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:46,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +30: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:46,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:46,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:46,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +20: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +10: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +10: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +17: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +17: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:46,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +16: [2023-05-13 09:38:46,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:46,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:46,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +20: [2023-05-13 09:38:46,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +16: [2023-05-13 09:38:46,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:46,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +13: [2023-05-13 09:38:46,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:46,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:46,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:46,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:46,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:46,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +13: [2023-05-13 09:38:46,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:46,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +13: [2023-05-13 09:38:46,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +13: [2023-05-13 09:38:46,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +13: [2023-05-13 09:38:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +28: [2023-05-13 09:38:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +28: [2023-05-13 09:38:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:46,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +17: [2023-05-13 09:38:46,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:46,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt... +17: [2023-05-13 09:38:46,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt... +12: [2023-05-13 09:38:46,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +19: [2023-05-13 09:38:46,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:46,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:46,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:46,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:46,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:46,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +20: [2023-05-13 09:38:46,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:46,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:46,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +13: [2023-05-13 09:38:46,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +12: [2023-05-13 09:38:46,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:46,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:46,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:46,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:46,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:46,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:46,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:46,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:46,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +30: [2023-05-13 09:38:46,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:46,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:46,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +17: [2023-05-13 09:38:46,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +16: [2023-05-13 09:38:46,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:46,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:46,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:46,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:46,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:46,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:46,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +28: [2023-05-13 09:38:46,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_00-model_states.pt. +10: [2023-05-13 09:38:46,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:46,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:46,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:46,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:46,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:46,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:46,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:46,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:46,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:46,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:46,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:46,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:46,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:46,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:46,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:46,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:46,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:46,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:46,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:46,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:46,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:46,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +11: [2023-05-13 09:38:46,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +11: [2023-05-13 09:38:46,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +11: [2023-05-13 09:38:46,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +11: [2023-05-13 09:38:46,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:46,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:46,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:46,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:46,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:46,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:46,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:46,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:46,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:46,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:46,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:46,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:46,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:46,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:46,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:46,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:46,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:46,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:46,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:46,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +27: [2023-05-13 09:38:46,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:46,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:46,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:46,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:46,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:46,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:46,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:46,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:46,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:46,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:46,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:46,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:46,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:46,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:46,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:46,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:46,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:46,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:46,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:46,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:46,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:46,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:46,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:46,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:46,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:46,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:46,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:46,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:46,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:46,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:46,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:46,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:46,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:46,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:46,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:46,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:46,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:46,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:46,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:46,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:46,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:46,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:46,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:46,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:46,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:46,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:46,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:46,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:46,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:46,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:46,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:46,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:46,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:46,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:47,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:47,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:47,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:47,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:47,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:47,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:47,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:47,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:47,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:47,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:47,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:47,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:47,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:47,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:47,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:47,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:47,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:47,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:47,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:47,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:47,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:47,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:47,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:47,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:47,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:47,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:47,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:47,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:47,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:47,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:47,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:47,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:47,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:47,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:47,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:47,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:47,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:47,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:47,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:47,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:47,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:47,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:47,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:47,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:47,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:47,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:47,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:47,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:47,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:47,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:47,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:47,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:47,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:47,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:47,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:47,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +14: [2023-05-13 09:38:47,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:47,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:47,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:47,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:47,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:47,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:47,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:47,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:47,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:47,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:47,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:47,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:47,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:47,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:47,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:47,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:47,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:47,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:47,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +14: [2023-05-13 09:38:47,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:47,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:47,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:47,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:47,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:47,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:47,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:47,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:47,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:47,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:47,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:47,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:47,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:47,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:47,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:47,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:47,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:47,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:47,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:47,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:47,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:47,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +14: [2023-05-13 09:38:47,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +14: [2023-05-13 09:38:47,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:47,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:47,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:47,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +14: [2023-05-13 09:38:47,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +30: [2023-05-13 09:38:47,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:47,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:47,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:47,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:47,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:47,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:47,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:47,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +21: [2023-05-13 09:38:47,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +21: [2023-05-13 09:38:47,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +21: [2023-05-13 09:38:47,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 3: [2023-05-13 09:38:47,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 3: [2023-05-13 09:38:47,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 3: [2023-05-13 09:38:47,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 3: [2023-05-13 09:38:47,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:47,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:47,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:47,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:47,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 5: [2023-05-13 09:38:47,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +22: [2023-05-13 09:38:47,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +15: [2023-05-13 09:38:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 5: [2023-05-13 09:38:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:47,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 5: [2023-05-13 09:38:47,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:47,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +15: [2023-05-13 09:38:47,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:47,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:47,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:47,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:47,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +15: [2023-05-13 09:38:47,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:47,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:47,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:47,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:47,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:47,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:47,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:47,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:47,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:47,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +18: [2023-05-13 09:38:47,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +18: [2023-05-13 09:38:47,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:47,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:47,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 0: [2023-05-13 09:38:47,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 5: [2023-05-13 09:38:47,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 5: [2023-05-13 09:38:47,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 5: [2023-05-13 09:38:47,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +18: [2023-05-13 09:38:47,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:47,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:47,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:47,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +15: [2023-05-13 09:38:47,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:47,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:47,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:47,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:47,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:47,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:47,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:47,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +18: [2023-05-13 09:38:47,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +22: [2023-05-13 09:38:47,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:47,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +22: [2023-05-13 09:38:47,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:47,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:47,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +22: [2023-05-13 09:38:47,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +15: [2023-05-13 09:38:47,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:47,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:47,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:47,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 0: [2023-05-13 09:38:47,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 0: [2023-05-13 09:38:47,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 0: [2023-05-13 09:38:47,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:47,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 0: [2023-05-13 09:38:47,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:47,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:47,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:47,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:47,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:47,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:47,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:47,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:47,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +18: [2023-05-13 09:38:47,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:47,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:47,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:47,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:47,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:47,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:47,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:47,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 5: [2023-05-13 09:38:47,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 5: [2023-05-13 09:38:47,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 5: [2023-05-13 09:38:47,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 3: [2023-05-13 09:38:47,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +18: [2023-05-13 09:38:47,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:47,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:47,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:47,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:47,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:47,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:47,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:47,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:47,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:47,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:47,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:47,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +26: [2023-05-13 09:38:47,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +26: [2023-05-13 09:38:47,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +26: [2023-05-13 09:38:47,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:47,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:47,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:47,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:47,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:47,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:47,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:47,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +17: [2023-05-13 09:38:47,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +30: [2023-05-13 09:38:47,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +30: [2023-05-13 09:38:47,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +30: [2023-05-13 09:38:47,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +30: [2023-05-13 09:38:47,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:47,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:47,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:47,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +16: [2023-05-13 09:38:47,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:47,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:47,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:47,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:47,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:47,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:47,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:47,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:47,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:47,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:47,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:47,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:47,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:47,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:47,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:47,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:47,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:47,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +20: [2023-05-13 09:38:47,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +20: [2023-05-13 09:38:47,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +20: [2023-05-13 09:38:47,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:47,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:47,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:47,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:47,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:47,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 9: [2023-05-13 09:38:47,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 7: [2023-05-13 09:38:47,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +23: [2023-05-13 09:38:47,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:47,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:47,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:47,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:47,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:47,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:47,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:47,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:47,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:47,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:47,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +24: [2023-05-13 09:38:47,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:47,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:47,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:47,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:47,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:47,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:47,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:47,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +23: [2023-05-13 09:38:47,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:47,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:47,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:47,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:47,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:47,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +20: [2023-05-13 09:38:47,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:47,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:47,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:47,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:47,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:47,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:47,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:47,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:47,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:47,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:47,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +17: [2023-05-13 09:38:47,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +29: [2023-05-13 09:38:47,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:47,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:47,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +28: [2023-05-13 09:38:47,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +29: [2023-05-13 09:38:47,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +10: [2023-05-13 09:38:47,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +10: [2023-05-13 09:38:47,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:47,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:47,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +29: [2023-05-13 09:38:47,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +10: [2023-05-13 09:38:47,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +26: [2023-05-13 09:38:47,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +29: [2023-05-13 09:38:47,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +26: [2023-05-13 09:38:47,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:47,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:47,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:47,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:47,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +30: [2023-05-13 09:38:47,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:47,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:47,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:47,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:47,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +26: [2023-05-13 09:38:47,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +26: [2023-05-13 09:38:47,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +30: [2023-05-13 09:38:47,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +30: [2023-05-13 09:38:47,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +24: [2023-05-13 09:38:47,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:47,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:47,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:47,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +17: [2023-05-13 09:38:47,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +17: [2023-05-13 09:38:47,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +17: [2023-05-13 09:38:47,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +30: [2023-05-13 09:38:47,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +30: [2023-05-13 09:38:47,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +10: [2023-05-13 09:38:47,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:47,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:47,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:47,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:47,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:47,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:47,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:47,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:47,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:47,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:47,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +20: [2023-05-13 09:38:47,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:47,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:47,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:47,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +25: [2023-05-13 09:38:47,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +25: [2023-05-13 09:38:47,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +25: [2023-05-13 09:38:47,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +20: [2023-05-13 09:38:47,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:47,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:47,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:47,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:47,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:47,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:47,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:47,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:47,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:47,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:47,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 7: [2023-05-13 09:38:47,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:47,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:47,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +20: [2023-05-13 09:38:47,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:47,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:47,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:47,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:47,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 9: [2023-05-13 09:38:47,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:47,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +19: [2023-05-13 09:38:47,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 7: [2023-05-13 09:38:47,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:47,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:47,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +19: [2023-05-13 09:38:47,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +28: [2023-05-13 09:38:47,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:47,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +28: [2023-05-13 09:38:47,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +20: [2023-05-13 09:38:47,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:47,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 1: [2023-05-13 09:38:47,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:47,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:47,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:47,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 6: [2023-05-13 09:38:47,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 6: [2023-05-13 09:38:47,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 6: [2023-05-13 09:38:47,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +27: [2023-05-13 09:38:47,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:47,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:47,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +28: [2023-05-13 09:38:47,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:47,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:47,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +27: [2023-05-13 09:38:47,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:47,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:47,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:47,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:47,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +31: [2023-05-13 09:38:47,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:47,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +27: [2023-05-13 09:38:47,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:47,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:47,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:47,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +25: [2023-05-13 09:38:47,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:47,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:47,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:47,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:47,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:47,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:47,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +24: [2023-05-13 09:38:47,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +24: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:47,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:47,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:47,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:47,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:47,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:47,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:47,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:47,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:47,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:47,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +31: [2023-05-13 09:38:47,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +24: [2023-05-13 09:38:47,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +24: [2023-05-13 09:38:47,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:47,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +11: [2023-05-13 09:38:47,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:47,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:47,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +13: [2023-05-13 09:38:47,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +13: [2023-05-13 09:38:47,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +23: [2023-05-13 09:38:47,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +23: [2023-05-13 09:38:47,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +13: [2023-05-13 09:38:47,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +23: [2023-05-13 09:38:47,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +23: [2023-05-13 09:38:47,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:47,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:47,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:47,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:47,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:47,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:47,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 2: [2023-05-13 09:38:47,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 4: [2023-05-13 09:38:47,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:47,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:47,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:47,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +12: [2023-05-13 09:38:47,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +11: [2023-05-13 09:38:47,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:47,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:47,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:47,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:47,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. + 8: [2023-05-13 09:38:47,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_04-model_01-model_states.pt. +25: [2023-05-13 09:38:47,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:47,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +11: [2023-05-13 09:38:47,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:47,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:47,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +19: [2023-05-13 09:38:47,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:47,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:47,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:47,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:47,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:47,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:47,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:47,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:47,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:47,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:47,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:47,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +23: [2023-05-13 09:38:47,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:47,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:47,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:47,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:47,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +25: [2023-05-13 09:38:47,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +25: [2023-05-13 09:38:47,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 8: [2023-05-13 09:38:47,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +19: [2023-05-13 09:38:47,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +19: [2023-05-13 09:38:47,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:47,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 1: [2023-05-13 09:38:47,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +12: [2023-05-13 09:38:47,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +11: [2023-05-13 09:38:47,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:47,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:47,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:47,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:47,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:47,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:47,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:47,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +16: [2023-05-13 09:38:47,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:47,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:47,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:47,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:47,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:47,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +31: [2023-05-13 09:38:47,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:47,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +31: [2023-05-13 09:38:47,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 6: [2023-05-13 09:38:47,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +16: [2023-05-13 09:38:47,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:47,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:47,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:47,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +16: [2023-05-13 09:38:47,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:47,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 4: [2023-05-13 09:38:47,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 4: [2023-05-13 09:38:47,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +23: [2023-05-13 09:38:47,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:47,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +23: [2023-05-13 09:38:47,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +23: [2023-05-13 09:38:47,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 8: [2023-05-13 09:38:47,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +13: [2023-05-13 09:38:47,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:47,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:47,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:47,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +23: [2023-05-13 09:38:47,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +23: [2023-05-13 09:38:47,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:47,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:47,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:47,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:47,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:47,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:47,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:47,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:47,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:47,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +13: [2023-05-13 09:38:47,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +13: [2023-05-13 09:38:47,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:47,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:47,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +23: [2023-05-13 09:38:47,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:47,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:47,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:47,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +21: [2023-05-13 09:38:47,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:47,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 8: [2023-05-13 09:38:47,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:47,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +21: [2023-05-13 09:38:47,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 8: [2023-05-13 09:38:47,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:47,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:47,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:47,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +12: [2023-05-13 09:38:47,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +14: [2023-05-13 09:38:47,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +14: [2023-05-13 09:38:47,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +14: [2023-05-13 09:38:47,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +14: [2023-05-13 09:38:47,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:47,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:47,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:47,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +21: [2023-05-13 09:38:47,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +14: [2023-05-13 09:38:47,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:47,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:47,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:47,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:47,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:47,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:47,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:47,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:47,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:47,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:47,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +22: [2023-05-13 09:38:47,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +22: [2023-05-13 09:38:47,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +22: [2023-05-13 09:38:47,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:47,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:47,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:47,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:47,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:47,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:47,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:47,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:47,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:47,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:47,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:47,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:47,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:47,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:47,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:47,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 3: [2023-05-13 09:38:47,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 3: [2023-05-13 09:38:47,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 3: [2023-05-13 09:38:47,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 1: [2023-05-13 09:38:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:47,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:47,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:47,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:47,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:47,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:47,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:47,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:47,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:47,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:47,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:47,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:47,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:47,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:47,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:47,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:47,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:47,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:47,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:47,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:47,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:47,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +18: [2023-05-13 09:38:47,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:47,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:47,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:47,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +18: [2023-05-13 09:38:47,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +18: [2023-05-13 09:38:47,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:47,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 0: [2023-05-13 09:38:47,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 0: [2023-05-13 09:38:47,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 0: [2023-05-13 09:38:47,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 6: [2023-05-13 09:38:47,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:47,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:47,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:47,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 5: [2023-05-13 09:38:47,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 0: [2023-05-13 09:38:47,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +18: [2023-05-13 09:38:47,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:47,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:47,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:47,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:47,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:47,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:47,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:47,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:47,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:47,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +29: [2023-05-13 09:38:47,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +29: [2023-05-13 09:38:47,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:47,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:47,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:47,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:47,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:47,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +27: [2023-05-13 09:38:47,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +27: [2023-05-13 09:38:47,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +27: [2023-05-13 09:38:47,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:47,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:47,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:47,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:47,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:47,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:47,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:47,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:47,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:47,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:47,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:47,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:47,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:47,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:47,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:47,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 2: [2023-05-13 09:38:47,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:47,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:47,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:47,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:47,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:47,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:47,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:47,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:47,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 2: [2023-05-13 09:38:47,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 2: [2023-05-13 09:38:47,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 2: [2023-05-13 09:38:47,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... + 2: [2023-05-13 09:38:47,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +20: [2023-05-13 09:38:47,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:47,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:47,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:47,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:47,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:47,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:47,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:47,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +17: [2023-05-13 09:38:47,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:47,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:47,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:47,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:47,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:47,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:47,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:47,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:47,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 7: [2023-05-13 09:38:47,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 7: [2023-05-13 09:38:47,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +27: [2023-05-13 09:38:47,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:47,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:47,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:47,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:47,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:47,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:47,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:47,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:47,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 9: [2023-05-13 09:38:47,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 9: [2023-05-13 09:38:47,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 9: [2023-05-13 09:38:47,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:47,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 9: [2023-05-13 09:38:47,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:47,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:47,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +17: [2023-05-13 09:38:47,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:47,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:47,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +14: [2023-05-13 09:38:47,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 2: [2023-05-13 09:38:47,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:47,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:47,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +15: [2023-05-13 09:38:47,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:47,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:47,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:47,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:47,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:47,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:47,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +14: [2023-05-13 09:38:47,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:47,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:47,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:47,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:47,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:47,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:47,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:47,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +31: [2023-05-13 09:38:47,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +31: [2023-05-13 09:38:47,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +27: [2023-05-13 09:38:47,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:47,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +17: [2023-05-13 09:38:47,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:47,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:47,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:47,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:47,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:47,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:47,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:47,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:47,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:47,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 4: [2023-05-13 09:38:47,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 4: [2023-05-13 09:38:47,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 4: [2023-05-13 09:38:47,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +15: [2023-05-13 09:38:47,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:47,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:47,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:47,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:47,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:47,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 8: [2023-05-13 09:38:47,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 8: [2023-05-13 09:38:47,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +15: [2023-05-13 09:38:47,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:47,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:47,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:47,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:47,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:47,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 0: [2023-05-13 09:38:47,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 8: [2023-05-13 09:38:47,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 4: [2023-05-13 09:38:47,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:47,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:47,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:47,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:47,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +15: [2023-05-13 09:38:47,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:47,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:47,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:47,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:47,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:47,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:47,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:47,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 1: [2023-05-13 09:38:47,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +29: [2023-05-13 09:38:47,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +29: [2023-05-13 09:38:47,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:47,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:47,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:47,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 7: [2023-05-13 09:38:47,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:47,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:47,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +29: [2023-05-13 09:38:47,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:47,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +17: [2023-05-13 09:38:47,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +29: [2023-05-13 09:38:47,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:47,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +26: [2023-05-13 09:38:47,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:47,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +29: [2023-05-13 09:38:47,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:47,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:47,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +26: [2023-05-13 09:38:47,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:47,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:47,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:47,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:47,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:47,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:47,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:47,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:47,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:47,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:47,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:47,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:47,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +30: [2023-05-13 09:38:47,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +30: [2023-05-13 09:38:47,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +30: [2023-05-13 09:38:47,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 1: [2023-05-13 09:38:47,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:47,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +29: [2023-05-13 09:38:47,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:47,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 5: [2023-05-13 09:38:47,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 5: [2023-05-13 09:38:47,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 5: [2023-05-13 09:38:47,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:47,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:47,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:47,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 7: [2023-05-13 09:38:47,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +26: [2023-05-13 09:38:47,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +26: [2023-05-13 09:38:47,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 9: [2023-05-13 09:38:47,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 9: [2023-05-13 09:38:47,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:47,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:47,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:47,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:47,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:47,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +30: [2023-05-13 09:38:47,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:47,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:47,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +30: [2023-05-13 09:38:47,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:47,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +24: [2023-05-13 09:38:47,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +24: [2023-05-13 09:38:47,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +29: [2023-05-13 09:38:47,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:47,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:47,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:47,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:47,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:47,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:47,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:47,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:47,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:47,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +29: [2023-05-13 09:38:47,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:47,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 0: [2023-05-13 09:38:47,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +28: [2023-05-13 09:38:47,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +29: [2023-05-13 09:38:47,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:47,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 1: [2023-05-13 09:38:47,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:47,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:47,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:47,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:47,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +30: [2023-05-13 09:38:47,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:47,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:47,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +28: [2023-05-13 09:38:47,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +28: [2023-05-13 09:38:47,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +15: [2023-05-13 09:38:47,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +15: [2023-05-13 09:38:47,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 1: [2023-05-13 09:38:47,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 7: [2023-05-13 09:38:47,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:47,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +15: [2023-05-13 09:38:47,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +29: [2023-05-13 09:38:47,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:47,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:47,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:47,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:47,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:47,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:47,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:47,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:47,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:47,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:47,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:47,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:47,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:47,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:47,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +15: [2023-05-13 09:38:47,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:47,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:47,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:47,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:47,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:47,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:47,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:47,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:47,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:47,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:47,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:47,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:47,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:47,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:47,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:47,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:47,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:47,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:47,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:47,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:47,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:47,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:47,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:47,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:47,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:47,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:47,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:47,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:47,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:47,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +22: [2023-05-13 09:38:47,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:47,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:47,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:47,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:47,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +26: [2023-05-13 09:38:47,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:47,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +22: [2023-05-13 09:38:47,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +28: [2023-05-13 09:38:47,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:47,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:47,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +28: [2023-05-13 09:38:47,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:47,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:47,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. + 9: [2023-05-13 09:38:47,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:47,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:47,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:47,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:47,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:47,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:47,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +26: [2023-05-13 09:38:47,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 9: [2023-05-13 09:38:47,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:47,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:47,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:47,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:47,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:47,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:47,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:47,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +13: [2023-05-13 09:38:47,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +26: [2023-05-13 09:38:47,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +13: [2023-05-13 09:38:47,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:47,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +10: [2023-05-13 09:38:47,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +10: [2023-05-13 09:38:47,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +10: [2023-05-13 09:38:47,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +13: [2023-05-13 09:38:47,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:47,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +10: [2023-05-13 09:38:47,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt... +26: [2023-05-13 09:38:47,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +13: [2023-05-13 09:38:47,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:47,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt... +20: [2023-05-13 09:38:47,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:47,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:47,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:47,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:47,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:47,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:47,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:47,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:47,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:47,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:47,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +15: [2023-05-13 09:38:47,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +15: [2023-05-13 09:38:47,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 6: [2023-05-13 09:38:47,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:47,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +15: [2023-05-13 09:38:47,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:47,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +25: [2023-05-13 09:38:47,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:47,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:47,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:47,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +21: [2023-05-13 09:38:47,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +28: [2023-05-13 09:38:47,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:47,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:47,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:47,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:47,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:47,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:47,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:47,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:47,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:47,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 6: [2023-05-13 09:38:47,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:47,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:47,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:47,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:47,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:47,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:47,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:47,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:47,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:47,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:47,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:47,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:47,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:47,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:47,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:47,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:47,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:47,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:47,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:47,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:47,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:47,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:47,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:47,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:47,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:47,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:47,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +13: [2023-05-13 09:38:47,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:47,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:47,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +13: [2023-05-13 09:38:47,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:47,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:47,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +13: [2023-05-13 09:38:47,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +12: [2023-05-13 09:38:47,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +12: [2023-05-13 09:38:47,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +10: [2023-05-13 09:38:47,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +10: [2023-05-13 09:38:47,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_00-model_states.pt. +12: [2023-05-13 09:38:47,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:47,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:47,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:47,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:47,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:47,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:47,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:47,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:47,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 4: [2023-05-13 09:38:47,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 4: [2023-05-13 09:38:47,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 4: [2023-05-13 09:38:47,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +31: [2023-05-13 09:38:47,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +31: [2023-05-13 09:38:47,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 3: [2023-05-13 09:38:47,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:47,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 3: [2023-05-13 09:38:47,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 3: [2023-05-13 09:38:47,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 3: [2023-05-13 09:38:47,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 3: [2023-05-13 09:38:47,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 3: [2023-05-13 09:38:47,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +31: [2023-05-13 09:38:47,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +31: [2023-05-13 09:38:47,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +31: [2023-05-13 09:38:47,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +22: [2023-05-13 09:38:47,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:47,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +19: [2023-05-13 09:38:47,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +22: [2023-05-13 09:38:47,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:47,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:47,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:47,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +20: [2023-05-13 09:38:47,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:47,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:47,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:47,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:47,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:47,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +22: [2023-05-13 09:38:47,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:47,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 6: [2023-05-13 09:38:47,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 6: [2023-05-13 09:38:47,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:47,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:47,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:47,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 6: [2023-05-13 09:38:47,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 2: [2023-05-13 09:38:47,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +11: [2023-05-13 09:38:47,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +11: [2023-05-13 09:38:47,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +11: [2023-05-13 09:38:47,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +11: [2023-05-13 09:38:47,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +11: [2023-05-13 09:38:47,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:47,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:47,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:47,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +16: [2023-05-13 09:38:47,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +23: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +23: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:47,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +25: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +25: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +25: [2023-05-13 09:38:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +25: [2023-05-13 09:38:47,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +11: [2023-05-13 09:38:47,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:47,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:47,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:47,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:47,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:47,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:47,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:47,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:47,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 8: [2023-05-13 09:38:47,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +16: [2023-05-13 09:38:47,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 8: [2023-05-13 09:38:47,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:47,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:47,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:47,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:47,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:47,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:47,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:47,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:47,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:47,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:47,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 1: [2023-05-13 09:38:47,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:47,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:47,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +13: [2023-05-13 09:38:47,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:47,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:47,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +21: [2023-05-13 09:38:47,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:47,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +21: [2023-05-13 09:38:47,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:47,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +21: [2023-05-13 09:38:47,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:47,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:47,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 2: [2023-05-13 09:38:47,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:47,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:47,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:47,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:47,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:47,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +27: [2023-05-13 09:38:47,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:47,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:47,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:47,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:47,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:47,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:47,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +16: [2023-05-13 09:38:47,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:47,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. + 6: [2023-05-13 09:38:47,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +16: [2023-05-13 09:38:47,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:47,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +25: [2023-05-13 09:38:47,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:47,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +24: [2023-05-13 09:38:47,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:47,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:47,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:47,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +24: [2023-05-13 09:38:47,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:47,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:47,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:47,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +11: [2023-05-13 09:38:47,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:47,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:47,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +25: [2023-05-13 09:38:47,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +12: [2023-05-13 09:38:48,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:48,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:48,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:48,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:48,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:48,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:48,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:48,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:48,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:48,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:48,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:48,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:48,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:48,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:48,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:48,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:48,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:48,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:48,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +23: [2023-05-13 09:38:48,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +19: [2023-05-13 09:38:48,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:48,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:48,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:48,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:48,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:48,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:48,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:48,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:48,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:48,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:48,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:48,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:48,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:48,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:48,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:48,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:48,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:48,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:48,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:48,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:48,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:48,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:48,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +19: [2023-05-13 09:38:48,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:48,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +13: [2023-05-13 09:38:48,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +16: [2023-05-13 09:38:48,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:48,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:48,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +13: [2023-05-13 09:38:48,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +13: [2023-05-13 09:38:48,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +19: [2023-05-13 09:38:48,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:48,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +19: [2023-05-13 09:38:48,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +19: [2023-05-13 09:38:48,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:48,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:48,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 6: [2023-05-13 09:38:48,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:48,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:48,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +12: [2023-05-13 09:38:48,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:48,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +19: [2023-05-13 09:38:48,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:48,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:48,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:48,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +16: [2023-05-13 09:38:48,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:48,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +16: [2023-05-13 09:38:48,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:48,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:48,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:48,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +25: [2023-05-13 09:38:48,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +24: [2023-05-13 09:38:48,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 8: [2023-05-13 09:38:48,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:48,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 8: [2023-05-13 09:38:48,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +16: [2023-05-13 09:38:48,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +16: [2023-05-13 09:38:48,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:48,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:48,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +14: [2023-05-13 09:38:48,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +14: [2023-05-13 09:38:48,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +25: [2023-05-13 09:38:48,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +25: [2023-05-13 09:38:48,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +11: [2023-05-13 09:38:48,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +24: [2023-05-13 09:38:48,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +14: [2023-05-13 09:38:48,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +11: [2023-05-13 09:38:48,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +11: [2023-05-13 09:38:48,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:48,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:48,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +23: [2023-05-13 09:38:48,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:48,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:48,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +20: [2023-05-13 09:38:48,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +20: [2023-05-13 09:38:48,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:48,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:48,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:48,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +20: [2023-05-13 09:38:48,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:48,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:48,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:48,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:48,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +23: [2023-05-13 09:38:48,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +24: [2023-05-13 09:38:48,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:48,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:48,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:48,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:48,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:48,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:48,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:48,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:48,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:48,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:48,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:48,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:48,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:48,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:48,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:48,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:48,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +23: [2023-05-13 09:38:48,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +17: [2023-05-13 09:38:48,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:48,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:48,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:48,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:48,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +23: [2023-05-13 09:38:48,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +20: [2023-05-13 09:38:48,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:48,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:48,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:48,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:48,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:48,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:48,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:48,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:48,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:48,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +14: [2023-05-13 09:38:48,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:48,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 7: [2023-05-13 09:38:48,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +17: [2023-05-13 09:38:48,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +14: [2023-05-13 09:38:48,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +14: [2023-05-13 09:38:48,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +14: [2023-05-13 09:38:48,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +17: [2023-05-13 09:38:48,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:48,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:48,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +18: [2023-05-13 09:38:48,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:48,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:48,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:48,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:48,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:48,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +18: [2023-05-13 09:38:48,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:48,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +20: [2023-05-13 09:38:48,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +20: [2023-05-13 09:38:48,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:48,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 2: [2023-05-13 09:38:48,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 2: [2023-05-13 09:38:48,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 2: [2023-05-13 09:38:48,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... + 2: [2023-05-13 09:38:48,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +20: [2023-05-13 09:38:48,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:48,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:48,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 5: [2023-05-13 09:38:48,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:48,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +10: [2023-05-13 09:38:48,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +18: [2023-05-13 09:38:48,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +10: [2023-05-13 09:38:48,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_05-model_01-model_states.pt. +14: [2023-05-13 09:38:48,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:48,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:48,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:48,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:48,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:48,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:48,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:48,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:48,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:48,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:48,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:48,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +17: [2023-05-13 09:38:48,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +14: [2023-05-13 09:38:48,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:48,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +14: [2023-05-13 09:38:48,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:48,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:48,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +17: [2023-05-13 09:38:48,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:48,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:48,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:48,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:48,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:48,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:48,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +29: [2023-05-13 09:38:48,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:48,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:48,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:48,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +17: [2023-05-13 09:38:48,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:48,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:48,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +18: [2023-05-13 09:38:48,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +18: [2023-05-13 09:38:48,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +18: [2023-05-13 09:38:48,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:48,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:48,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:48,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +22: [2023-05-13 09:38:48,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:48,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:48,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 5: [2023-05-13 09:38:48,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 5: [2023-05-13 09:38:48,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 5: [2023-05-13 09:38:48,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:48,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:48,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:48,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:48,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:48,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:48,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:48,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +14: [2023-05-13 09:38:48,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:48,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +27: [2023-05-13 09:38:48,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:48,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +24: [2023-05-13 09:38:48,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +24: [2023-05-13 09:38:48,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 2: [2023-05-13 09:38:48,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:48,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:48,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:48,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:48,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... +10: [2023-05-13 09:38:48,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:48,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:48,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:48,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:48,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:48,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:48,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:48,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:48,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:48,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:48,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:48,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:48,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:48,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:48,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:48,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +10: [2023-05-13 09:38:48,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +30: [2023-05-13 09:38:48,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:48,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:48,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +10: [2023-05-13 09:38:48,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +10: [2023-05-13 09:38:48,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +24: [2023-05-13 09:38:48,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:48,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +31: [2023-05-13 09:38:48,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +24: [2023-05-13 09:38:48,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +24: [2023-05-13 09:38:48,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:48,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +31: [2023-05-13 09:38:48,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +26: [2023-05-13 09:38:48,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:48,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:48,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt... +27: [2023-05-13 09:38:48,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:48,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:48,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:48,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:48,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 1: [2023-05-13 09:38:48,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 1: [2023-05-13 09:38:48,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 1: [2023-05-13 09:38:48,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +27: [2023-05-13 09:38:48,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:48,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +27: [2023-05-13 09:38:48,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +26: [2023-05-13 09:38:48,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_00-model_states.pt. +31: [2023-05-13 09:38:48,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:48,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:48,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:48,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:48,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +27: [2023-05-13 09:38:48,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:48,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:48,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:48,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +18: [2023-05-13 09:38:48,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:48,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:48,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:48,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +18: [2023-05-13 09:38:48,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:48,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:48,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +27: [2023-05-13 09:38:48,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:48,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:48,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:48,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:48,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +18: [2023-05-13 09:38:48,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:48,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:48,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:48,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:48,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:48,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:48,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:48,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:48,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +26: [2023-05-13 09:38:48,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +26: [2023-05-13 09:38:48,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +12: [2023-05-13 09:38:48,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +12: [2023-05-13 09:38:48,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:48,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:48,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:48,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +12: [2023-05-13 09:38:48,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +30: [2023-05-13 09:38:48,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:48,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:48,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 5: [2023-05-13 09:38:48,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 5: [2023-05-13 09:38:48,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +26: [2023-05-13 09:38:48,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 5: [2023-05-13 09:38:48,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +30: [2023-05-13 09:38:48,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:48,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:48,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:48,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +30: [2023-05-13 09:38:48,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +28: [2023-05-13 09:38:48,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:48,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:48,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:48,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:48,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:48,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:48,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:48,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +28: [2023-05-13 09:38:48,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:48,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:48,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:48,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 8: [2023-05-13 09:38:48,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:48,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:48,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:48,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:48,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +30: [2023-05-13 09:38:48,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:48,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:48,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:48,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:48,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:48,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:48,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:48,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:48,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:48,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:48,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +22: [2023-05-13 09:38:48,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:48,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:48,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:48,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:48,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:48,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:48,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:48,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +22: [2023-05-13 09:38:48,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +22: [2023-05-13 09:38:48,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:48,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +30: [2023-05-13 09:38:48,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:48,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:48,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +30: [2023-05-13 09:38:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:48,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:48,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +19: [2023-05-13 09:38:48,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:48,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:48,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 3: [2023-05-13 09:38:48,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:48,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:48,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:48,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 0: [2023-05-13 09:38:48,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:48,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:48,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 8: [2023-05-13 09:38:48,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +15: [2023-05-13 09:38:48,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:48,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:48,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:48,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:48,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:48,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:48,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:48,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:48,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:48,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:48,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:48,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:48,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:48,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:48,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:48,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:48,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:48,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:48,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:48,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:48,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:48,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:48,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:48,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:48,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:48,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:48,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:48,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +29: [2023-05-13 09:38:48,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +29: [2023-05-13 09:38:48,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +29: [2023-05-13 09:38:48,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:48,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:48,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:48,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:48,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:48,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:48,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:48,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:48,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:48,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 6: [2023-05-13 09:38:48,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +26: [2023-05-13 09:38:48,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:48,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +31: [2023-05-13 09:38:48,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:48,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:48,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:48,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:48,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +31: [2023-05-13 09:38:48,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +31: [2023-05-13 09:38:48,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:48,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:48,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:48,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:48,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +26: [2023-05-13 09:38:48,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:48,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:48,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:48,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:48,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +26: [2023-05-13 09:38:48,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 4: [2023-05-13 09:38:48,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:48,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +28: [2023-05-13 09:38:48,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +31: [2023-05-13 09:38:48,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:48,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:48,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:48,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:48,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:48,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 3: [2023-05-13 09:38:48,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 8: [2023-05-13 09:38:48,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +22: [2023-05-13 09:38:48,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:48,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:48,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:48,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:48,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:48,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:48,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:48,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 6: [2023-05-13 09:38:48,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 6: [2023-05-13 09:38:48,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 8: [2023-05-13 09:38:48,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:48,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +25: [2023-05-13 09:38:48,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 6: [2023-05-13 09:38:48,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +25: [2023-05-13 09:38:48,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +13: [2023-05-13 09:38:48,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +13: [2023-05-13 09:38:48,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 8: [2023-05-13 09:38:48,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +22: [2023-05-13 09:38:48,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:48,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +22: [2023-05-13 09:38:48,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:48,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:48,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +25: [2023-05-13 09:38:48,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +13: [2023-05-13 09:38:48,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +21: [2023-05-13 09:38:48,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 8: [2023-05-13 09:38:48,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:48,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:48,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +11: [2023-05-13 09:38:48,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 8: [2023-05-13 09:38:48,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 2: [2023-05-13 09:38:48,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:48,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +15: [2023-05-13 09:38:48,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +11: [2023-05-13 09:38:48,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 9: [2023-05-13 09:38:48,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +28: [2023-05-13 09:38:48,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +22: [2023-05-13 09:38:48,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:48,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +15: [2023-05-13 09:38:48,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:48,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:48,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +23: [2023-05-13 09:38:48,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:48,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:48,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:48,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +16: [2023-05-13 09:38:48,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 8: [2023-05-13 09:38:48,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +15: [2023-05-13 09:38:48,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:48,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:48,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:48,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:48,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 9: [2023-05-13 09:38:48,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:48,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:48,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:48,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 3: [2023-05-13 09:38:48,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:48,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +19: [2023-05-13 09:38:48,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +19: [2023-05-13 09:38:48,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 9: [2023-05-13 09:38:48,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:48,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:48,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:48,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +28: [2023-05-13 09:38:48,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 3: [2023-05-13 09:38:48,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:48,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:48,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:48,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +22: [2023-05-13 09:38:48,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:48,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:48,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:48,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:48,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:48,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +21: [2023-05-13 09:38:48,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:48,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:48,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:48,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:48,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:48,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:48,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:48,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:48,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:48,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +28: [2023-05-13 09:38:48,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:48,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:48,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:48,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:48,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:48,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:48,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:48,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:48,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:48,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:48,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:48,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:48,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:48,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:48,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:48,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:48,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:48,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:48,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:48,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +16: [2023-05-13 09:38:48,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:48,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +29: [2023-05-13 09:38:48,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:48,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:48,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:48,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:48,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:48,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 8: [2023-05-13 09:38:48,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:48,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +29: [2023-05-13 09:38:48,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:48,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:48,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:48,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:48,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:48,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. +10: [2023-05-13 09:38:48,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_06-model_01-model_states.pt. + 4: [2023-05-13 09:38:48,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +15: [2023-05-13 09:38:48,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:48,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:48,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +31: [2023-05-13 09:38:48,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +31: [2023-05-13 09:38:48,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +15: [2023-05-13 09:38:48,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +15: [2023-05-13 09:38:48,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +15: [2023-05-13 09:38:48,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:48,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:48,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 4: [2023-05-13 09:38:48,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 4: [2023-05-13 09:38:48,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 4: [2023-05-13 09:38:48,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +20: [2023-05-13 09:38:48,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:48,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:48,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 6: [2023-05-13 09:38:48,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:48,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:48,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:48,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:48,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:48,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +13: [2023-05-13 09:38:48,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 3: [2023-05-13 09:38:48,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:48,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:48,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:48,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:48,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:48,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:48,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:48,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:48,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +21: [2023-05-13 09:38:48,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:48,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +21: [2023-05-13 09:38:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 8: [2023-05-13 09:38:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:48,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:48,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 3: [2023-05-13 09:38:48,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +24: [2023-05-13 09:38:48,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +31: [2023-05-13 09:38:48,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:48,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:48,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:48,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:48,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +11: [2023-05-13 09:38:48,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:48,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:48,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:48,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:48,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +28: [2023-05-13 09:38:48,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +18: [2023-05-13 09:38:48,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +25: [2023-05-13 09:38:48,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:48,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:48,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +23: [2023-05-13 09:38:48,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:48,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:48,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:48,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:48,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:48,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:48,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:48,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:48,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:48,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:48,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:48,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:48,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +12: [2023-05-13 09:38:48,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:48,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:48,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:48,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:48,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:48,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:48,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:48,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:48,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:48,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +12: [2023-05-13 09:38:48,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:48,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:48,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:48,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +25: [2023-05-13 09:38:48,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:48,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:48,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +12: [2023-05-13 09:38:48,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:48,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +25: [2023-05-13 09:38:48,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:48,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +23: [2023-05-13 09:38:48,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:48,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:48,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:48,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +11: [2023-05-13 09:38:48,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:48,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:48,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:48,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +16: [2023-05-13 09:38:48,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +23: [2023-05-13 09:38:48,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 1: [2023-05-13 09:38:48,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +11: [2023-05-13 09:38:48,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 4: [2023-05-13 09:38:48,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:48,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:48,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +13: [2023-05-13 09:38:48,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:48,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +20: [2023-05-13 09:38:48,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +20: [2023-05-13 09:38:48,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +20: [2023-05-13 09:38:48,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +24: [2023-05-13 09:38:48,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:48,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +25: [2023-05-13 09:38:48,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:48,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:48,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 4: [2023-05-13 09:38:48,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:48,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:48,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:48,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:48,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:48,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +13: [2023-05-13 09:38:48,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... + 2: [2023-05-13 09:38:48,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:48,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:48,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:48,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:48,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:48,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:48,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:48,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:48,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:48,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +16: [2023-05-13 09:38:48,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +14: [2023-05-13 09:38:48,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:48,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:48,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:48,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:48,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:48,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:48,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:48,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:48,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +24: [2023-05-13 09:38:48,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:48,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:48,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +18: [2023-05-13 09:38:48,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:48,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:48,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:48,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt... +10: [2023-05-13 09:38:48,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:48,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +10: [2023-05-13 09:38:48,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt... +17: [2023-05-13 09:38:48,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:48,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:48,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:48,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +16: [2023-05-13 09:38:48,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:48,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +20: [2023-05-13 09:38:48,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +20: [2023-05-13 09:38:48,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:48,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:48,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:48,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +17: [2023-05-13 09:38:48,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:48,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:48,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:48,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:48,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:48,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +10: [2023-05-13 09:38:48,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:48,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:48,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:48,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_00-model_states.pt. +19: [2023-05-13 09:38:48,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:48,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:48,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:48,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:48,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +19: [2023-05-13 09:38:48,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:48,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:48,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:48,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:48,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:48,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +17: [2023-05-13 09:38:48,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:48,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:48,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:48,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:48,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:48,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:48,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +27: [2023-05-13 09:38:48,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:48,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:48,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:48,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:48,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:48,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +17: [2023-05-13 09:38:48,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +17: [2023-05-13 09:38:48,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +26: [2023-05-13 09:38:48,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:48,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:48,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:48,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:48,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +17: [2023-05-13 09:38:48,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:48,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:48,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:48,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:48,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +18: [2023-05-13 09:38:48,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:48,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:48,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:48,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:48,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:48,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:48,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:48,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:48,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:48,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:48,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:48,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:48,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:48,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:48,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:48,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:48,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:48,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:48,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:48,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:48,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:48,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:48,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +26: [2023-05-13 09:38:48,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +22: [2023-05-13 09:38:48,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:48,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:48,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +26: [2023-05-13 09:38:48,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +27: [2023-05-13 09:38:48,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:48,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:48,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +27: [2023-05-13 09:38:48,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:48,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +26: [2023-05-13 09:38:48,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:48,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 5: [2023-05-13 09:38:48,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 5: [2023-05-13 09:38:48,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:48,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:48,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:48,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:48,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 7: [2023-05-13 09:38:48,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 7: [2023-05-13 09:38:48,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 7: [2023-05-13 09:38:48,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +26: [2023-05-13 09:38:48,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:48,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:48,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +26: [2023-05-13 09:38:48,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:48,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:48,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +27: [2023-05-13 09:38:48,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +27: [2023-05-13 09:38:48,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:48,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +27: [2023-05-13 09:38:48,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:48,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +26: [2023-05-13 09:38:48,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:48,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:48,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:48,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:48,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:48,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:48,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +26: [2023-05-13 09:38:48,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 0: [2023-05-13 09:38:48,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 7: [2023-05-13 09:38:48,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:48,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:48,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:48,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:48,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +24: [2023-05-13 09:38:48,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 9: [2023-05-13 09:38:48,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 9: [2023-05-13 09:38:48,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:48,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:48,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:48,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:48,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:48,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 5: [2023-05-13 09:38:48,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 5: [2023-05-13 09:38:48,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:48,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:48,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +12: [2023-05-13 09:38:48,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:48,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:48,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:48,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 3: [2023-05-13 09:38:48,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:48,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:48,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:48,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:48,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:48,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 0: [2023-05-13 09:38:48,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 0: [2023-05-13 09:38:48,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:48,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:48,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:48,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:48,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:48,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:48,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:48,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:48,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:48,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:48,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +24: [2023-05-13 09:38:48,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:48,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:48,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +24: [2023-05-13 09:38:48,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +31: [2023-05-13 09:38:48,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 0: [2023-05-13 09:38:48,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:48,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:48,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:48,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:48,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:48,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:48,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +21: [2023-05-13 09:38:48,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +31: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:48,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:48,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +31: [2023-05-13 09:38:48,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:48,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:48,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:48,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:48,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:48,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:48,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:48,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:48,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:48,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:48,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +14: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +28: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:48,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:48,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:48,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:48,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:48,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:48,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:48,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:48,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +31: [2023-05-13 09:38:48,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 4: [2023-05-13 09:38:48,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:48,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +11: [2023-05-13 09:38:48,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +11: [2023-05-13 09:38:48,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:48,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:48,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:48,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:48,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:48,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 6: [2023-05-13 09:38:48,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 6: [2023-05-13 09:38:48,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:48,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +25: [2023-05-13 09:38:48,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +25: [2023-05-13 09:38:48,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +25: [2023-05-13 09:38:48,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +25: [2023-05-13 09:38:48,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:48,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:48,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:48,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 1: [2023-05-13 09:38:48,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 1: [2023-05-13 09:38:48,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 1: [2023-05-13 09:38:48,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 1: [2023-05-13 09:38:48,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:48,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 2: [2023-05-13 09:38:48,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 2: [2023-05-13 09:38:48,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 2: [2023-05-13 09:38:48,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 2: [2023-05-13 09:38:48,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +23: [2023-05-13 09:38:48,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +23: [2023-05-13 09:38:48,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +23: [2023-05-13 09:38:48,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +23: [2023-05-13 09:38:48,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +19: [2023-05-13 09:38:48,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +19: [2023-05-13 09:38:48,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +19: [2023-05-13 09:38:48,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:48,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:48,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:48,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +19: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +30: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +15: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 3: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 8: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 8: [2023-05-13 09:38:48,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 8: [2023-05-13 09:38:48,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 8: [2023-05-13 09:38:48,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 9: [2023-05-13 09:38:48,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 9: [2023-05-13 09:38:48,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 9: [2023-05-13 09:38:48,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 8: [2023-05-13 09:38:48,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:48,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:48,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +14: [2023-05-13 09:38:48,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:48,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:48,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:48,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:48,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:48,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:48,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:48,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:48,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:48,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:48,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +14: [2023-05-13 09:38:48,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:48,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:48,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:48,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:48,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:48,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:48,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +12: [2023-05-13 09:38:48,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +12: [2023-05-13 09:38:48,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +12: [2023-05-13 09:38:48,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:48,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:48,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:48,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +11: [2023-05-13 09:38:48,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +12: [2023-05-13 09:38:48,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:48,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:48,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:48,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:48,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:48,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:48,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:48,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:48,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +11: [2023-05-13 09:38:48,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:48,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:48,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:48,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:48,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:48,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:48,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:48,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:48,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 3: [2023-05-13 09:38:48,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +23: [2023-05-13 09:38:48,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:48,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:48,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:48,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:48,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +15: [2023-05-13 09:38:48,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +22: [2023-05-13 09:38:48,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:48,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:48,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:48,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:48,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:48,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:48,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +21: [2023-05-13 09:38:48,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +21: [2023-05-13 09:38:48,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:48,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:48,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:48,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:48,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:48,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:48,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:48,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:48,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:48,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:48,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +30: [2023-05-13 09:38:48,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:48,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:48,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +21: [2023-05-13 09:38:48,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:48,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:48,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:48,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +14: [2023-05-13 09:38:48,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:48,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:48,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +21: [2023-05-13 09:38:48,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:48,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:48,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:48,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:48,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:48,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:48,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:48,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:48,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:48,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:48,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:48,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:48,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:48,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:48,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:48,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:48,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:48,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:48,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:48,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:48,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:48,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:48,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:48,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +28: [2023-05-13 09:38:48,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:48,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:48,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +19: [2023-05-13 09:38:48,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:48,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:48,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:48,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:48,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:48,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:48,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:48,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:48,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +19: [2023-05-13 09:38:48,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:48,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 1: [2023-05-13 09:38:48,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:48,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:48,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:48,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +19: [2023-05-13 09:38:48,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 6: [2023-05-13 09:38:48,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +12: [2023-05-13 09:38:48,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:48,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:48,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:48,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:48,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 1: [2023-05-13 09:38:48,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:48,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 6: [2023-05-13 09:38:48,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 3: [2023-05-13 09:38:48,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +13: [2023-05-13 09:38:48,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:48,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:48,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:48,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +11: [2023-05-13 09:38:48,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:48,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:48,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:48,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:48,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:48,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 3: [2023-05-13 09:38:48,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:48,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:48,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +29: [2023-05-13 09:38:48,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 3: [2023-05-13 09:38:48,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:48,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:48,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 9: [2023-05-13 09:38:48,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +13: [2023-05-13 09:38:48,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:48,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +11: [2023-05-13 09:38:48,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +29: [2023-05-13 09:38:48,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:48,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:48,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:48,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 8: [2023-05-13 09:38:48,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:48,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:48,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:48,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 3: [2023-05-13 09:38:48,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 3: [2023-05-13 09:38:48,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +13: [2023-05-13 09:38:48,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:48,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:48,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:48,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 8: [2023-05-13 09:38:48,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +29: [2023-05-13 09:38:48,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:48,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:48,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 8: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +28: [2023-05-13 09:38:48,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +28: [2023-05-13 09:38:48,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +28: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. + 4: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +23: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +28: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 4: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +23: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +22: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 1: [2023-05-13 09:38:48,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 9: [2023-05-13 09:38:48,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +15: [2023-05-13 09:38:48,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:48,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:48,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +30: [2023-05-13 09:38:48,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:48,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +15: [2023-05-13 09:38:48,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:48,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:48,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:48,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 9: [2023-05-13 09:38:48,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:48,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:48,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:48,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +30: [2023-05-13 09:38:48,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:48,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:48,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:48,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:48,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 4: [2023-05-13 09:38:48,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 9: [2023-05-13 09:38:48,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +12: [2023-05-13 09:38:48,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +20: [2023-05-13 09:38:48,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:48,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:48,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +12: [2023-05-13 09:38:48,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:48,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:48,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:48,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 4: [2023-05-13 09:38:48,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +20: [2023-05-13 09:38:48,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +20: [2023-05-13 09:38:48,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:48,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:48,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +25: [2023-05-13 09:38:48,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:48,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:48,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:48,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:48,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:48,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +16: [2023-05-13 09:38:48,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +16: [2023-05-13 09:38:48,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +13: [2023-05-13 09:38:48,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:48,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:48,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:48,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +16: [2023-05-13 09:38:48,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:48,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:48,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:48,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +10: [2023-05-13 09:38:48,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +10: [2023-05-13 09:38:48,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +10: [2023-05-13 09:38:48,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_07-model_01-model_states.pt. +29: [2023-05-13 09:38:48,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:48,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:48,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:48,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:48,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +22: [2023-05-13 09:38:48,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:48,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:48,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:48,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +24: [2023-05-13 09:38:48,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:48,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:48,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:48,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:48,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:48,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:48,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:48,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +17: [2023-05-13 09:38:48,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:48,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:48,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +29: [2023-05-13 09:38:48,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:48,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:48,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:48,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:48,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:48,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:48,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 1: [2023-05-13 09:38:48,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:48,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:48,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:48,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +18: [2023-05-13 09:38:48,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:48,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:48,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +25: [2023-05-13 09:38:48,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:48,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +18: [2023-05-13 09:38:48,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +18: [2023-05-13 09:38:48,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +18: [2023-05-13 09:38:48,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +29: [2023-05-13 09:38:48,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:48,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:48,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:48,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:48,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +16: [2023-05-13 09:38:48,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +10: [2023-05-13 09:38:48,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:48,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +29: [2023-05-13 09:38:48,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:48,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +25: [2023-05-13 09:38:48,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:48,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 2: [2023-05-13 09:38:48,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 2: [2023-05-13 09:38:48,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 2: [2023-05-13 09:38:48,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 2: [2023-05-13 09:38:48,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt... +13: [2023-05-13 09:38:48,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:48,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:48,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:48,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:48,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:48,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:48,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:48,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:48,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +24: [2023-05-13 09:38:48,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:48,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:48,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:48,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:48,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +13: [2023-05-13 09:38:48,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:48,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +28: [2023-05-13 09:38:48,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +20: [2023-05-13 09:38:48,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:48,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:48,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:48,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:48,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:48,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:48,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +28: [2023-05-13 09:38:48,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +13: [2023-05-13 09:38:48,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:48,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +13: [2023-05-13 09:38:48,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:48,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +20: [2023-05-13 09:38:48,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +20: [2023-05-13 09:38:48,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +20: [2023-05-13 09:38:48,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:48,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... + 6: [2023-05-13 09:38:48,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:48,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:48,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:48,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +19: [2023-05-13 09:38:48,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:48,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:48,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:48,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +16: [2023-05-13 09:38:48,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +13: [2023-05-13 09:38:48,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +24: [2023-05-13 09:38:48,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:48,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:48,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:48,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:48,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:48,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +10: [2023-05-13 09:38:48,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:48,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +16: [2023-05-13 09:38:48,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:48,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:48,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:48,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:48,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:48,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:48,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +17: [2023-05-13 09:38:48,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +24: [2023-05-13 09:38:48,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:48,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:48,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +26: [2023-05-13 09:38:48,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:48,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +16: [2023-05-13 09:38:48,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +26: [2023-05-13 09:38:48,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:48,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:48,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:48,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:48,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:48,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:48,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +10: [2023-05-13 09:38:48,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +18: [2023-05-13 09:38:48,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:48,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +24: [2023-05-13 09:38:48,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:48,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:48,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +10: [2023-05-13 09:38:48,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +10: [2023-05-13 09:38:48,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +10: [2023-05-13 09:38:48,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt... +18: [2023-05-13 09:38:48,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:48,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:48,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_00-model_states.pt. +17: [2023-05-13 09:38:48,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:48,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +18: [2023-05-13 09:38:48,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +18: [2023-05-13 09:38:48,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:48,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:48,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:48,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:48,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:48,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:48,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:48,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +17: [2023-05-13 09:38:48,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:48,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:48,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:48,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:48,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:48,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:48,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:48,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:48,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:48,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:48,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:48,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:48,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:48,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:48,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:48,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:48,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:48,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:48,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:48,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:48,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:48,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:48,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:48,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:48,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:48,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:48,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:48,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:48,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:48,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +17: [2023-05-13 09:38:48,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +17: [2023-05-13 09:38:48,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:48,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:48,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:48,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:48,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:48,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:48,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:48,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:48,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:48,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:48,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:48,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:48,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:48,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +31: [2023-05-13 09:38:48,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:48,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:48,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:48,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:48,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:48,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:48,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:48,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:48,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:48,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:48,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:48,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +26: [2023-05-13 09:38:48,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:48,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:48,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:48,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:48,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:48,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:48,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:48,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:48,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:48,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +27: [2023-05-13 09:38:48,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:48,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +31: [2023-05-13 09:38:48,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:48,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:48,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:48,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:48,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:48,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:48,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:48,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:48,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:48,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:48,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +24: [2023-05-13 09:38:48,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +24: [2023-05-13 09:38:48,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:48,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:48,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:48,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:48,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:48,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:48,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:48,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:48,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:48,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:48,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:48,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:48,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:48,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:48,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:48,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:48,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:48,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +24: [2023-05-13 09:38:48,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:48,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:48,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:48,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:48,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:48,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:48,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:48,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:48,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:48,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:48,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:48,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:48,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:48,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:48,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:48,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:48,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:48,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:48,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:48,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:48,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:48,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:48,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:48,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +31: [2023-05-13 09:38:48,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +31: [2023-05-13 09:38:48,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:48,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 7: [2023-05-13 09:38:48,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:48,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:48,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:48,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:48,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:48,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:48,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:48,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:48,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:48,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:49,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:49,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:49,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:49,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:49,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:49,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +24: [2023-05-13 09:38:49,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:49,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:49,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +24: [2023-05-13 09:38:49,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +23: [2023-05-13 09:38:49,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:49,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:49,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:49,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:49,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:49,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +14: [2023-05-13 09:38:49,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +18: [2023-05-13 09:38:49,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:49,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +14: [2023-05-13 09:38:49,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +23: [2023-05-13 09:38:49,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:49,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:49,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:49,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:49,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:49,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:49,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:49,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:49,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:49,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:49,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:49,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:49,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:49,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:49,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:49,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:49,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:49,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:49,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:49,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:49,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:49,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:49,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:49,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:49,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:49,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:49,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:49,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:49,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:49,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:49,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:49,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:49,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:49,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:49,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +24: [2023-05-13 09:38:49,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:49,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:49,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:49,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:49,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +24: [2023-05-13 09:38:49,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +29: [2023-05-13 09:38:49,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:49,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +14: [2023-05-13 09:38:49,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:49,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:49,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:49,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +14: [2023-05-13 09:38:49,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:49,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +25: [2023-05-13 09:38:49,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +12: [2023-05-13 09:38:49,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +12: [2023-05-13 09:38:49,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +12: [2023-05-13 09:38:49,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:49,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:49,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:49,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:49,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:49,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:49,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:49,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:49,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:49,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:49,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:49,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:49,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:49,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:49,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:49,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:49,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:49,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:49,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:49,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:49,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +20: [2023-05-13 09:38:49,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:49,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:49,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:49,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:49,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:49,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:49,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:49,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:49,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:49,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:49,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:49,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:49,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:49,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:49,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:49,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:49,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:49,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:49,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:49,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:49,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:49,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:49,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:49,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:49,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 0: [2023-05-13 09:38:49,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:49,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:49,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +30: [2023-05-13 09:38:49,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:49,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:49,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +27: [2023-05-13 09:38:49,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:49,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +21: [2023-05-13 09:38:49,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +30: [2023-05-13 09:38:49,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:49,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:49,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 0: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:49,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 0: [2023-05-13 09:38:49,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:49,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:49,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:49,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +22: [2023-05-13 09:38:49,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:49,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +20: [2023-05-13 09:38:49,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:49,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:49,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:49,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:49,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:49,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +26: [2023-05-13 09:38:49,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:49,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:49,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:49,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:49,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:49,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:49,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:49,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:49,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:49,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:49,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:49,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:49,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:49,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:49,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +21: [2023-05-13 09:38:49,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:49,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:49,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +18: [2023-05-13 09:38:49,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:49,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:49,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:49,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:49,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:49,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:49,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:49,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:49,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:49,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:49,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:49,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:49,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:49,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:49,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:49,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:49,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:49,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:49,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:49,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:49,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:49,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:49,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:49,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:49,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:49,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +26: [2023-05-13 09:38:49,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:49,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:49,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:49,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:49,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +31: [2023-05-13 09:38:49,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:49,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:49,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:49,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:49,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +12: [2023-05-13 09:38:49,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +12: [2023-05-13 09:38:49,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:49,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +12: [2023-05-13 09:38:49,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:49,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +27: [2023-05-13 09:38:49,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:49,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:49,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:49,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:49,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:49,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:49,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:49,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:49,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:49,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +20: [2023-05-13 09:38:49,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:49,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +20: [2023-05-13 09:38:49,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +20: [2023-05-13 09:38:49,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:49,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:49,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +18: [2023-05-13 09:38:49,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:49,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +18: [2023-05-13 09:38:49,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +30: [2023-05-13 09:38:49,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:49,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:49,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:49,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:49,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:49,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:49,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:49,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:49,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:49,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:49,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:49,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:49,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:49,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:49,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:49,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +18: [2023-05-13 09:38:49,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +18: [2023-05-13 09:38:49,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:49,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:49,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:49,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:49,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:49,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:49,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:49,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:49,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:49,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:49,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:49,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +12: [2023-05-13 09:38:49,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:49,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:49,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +12: [2023-05-13 09:38:49,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:49,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:49,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:49,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:49,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:49,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:49,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:49,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:49,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:49,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:49,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:49,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:49,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:49,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:49,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:49,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:49,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:49,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:49,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:49,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:49,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:49,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:49,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:49,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:49,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:49,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:49,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:49,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:49,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:49,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:49,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:49,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:49,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 6: [2023-05-13 09:38:49,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:49,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:49,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:49,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:49,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:49,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:49,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:49,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:49,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +29: [2023-05-13 09:38:49,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:49,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:49,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +29: [2023-05-13 09:38:49,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +22: [2023-05-13 09:38:49,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:49,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:49,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:49,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:49,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +22: [2023-05-13 09:38:49,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +12: [2023-05-13 09:38:49,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:49,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:49,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:49,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:49,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:49,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:49,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:49,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +19: [2023-05-13 09:38:49,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:49,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:49,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:49,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:49,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:49,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:49,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:49,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:49,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:49,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +12: [2023-05-13 09:38:49,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:49,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:49,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:49,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:49,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:49,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:49,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:49,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +13: [2023-05-13 09:38:49,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +13: [2023-05-13 09:38:49,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +13: [2023-05-13 09:38:49,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +13: [2023-05-13 09:38:49,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:49,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:49,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:49,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:49,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:49,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +30: [2023-05-13 09:38:49,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +30: [2023-05-13 09:38:49,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:49,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:49,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:49,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:49,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:49,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:49,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:49,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:49,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:49,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:49,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:49,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:49,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:49,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:49,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:49,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +21: [2023-05-13 09:38:49,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:49,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:49,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:49,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:49,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:49,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:49,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +26: [2023-05-13 09:38:49,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:49,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:49,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:49,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:49,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:49,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:49,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +26: [2023-05-13 09:38:49,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:49,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +11: [2023-05-13 09:38:49,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +11: [2023-05-13 09:38:49,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +11: [2023-05-13 09:38:49,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:49,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:49,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:49,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 3: [2023-05-13 09:38:49,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:49,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:49,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 3: [2023-05-13 09:38:49,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:49,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:49,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:49,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:49,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:49,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:49,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +23: [2023-05-13 09:38:49,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:49,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:49,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +19: [2023-05-13 09:38:49,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:49,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:49,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:49,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +10: [2023-05-13 09:38:49,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +19: [2023-05-13 09:38:49,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:49,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:49,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +13: [2023-05-13 09:38:49,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +30: [2023-05-13 09:38:49,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:49,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:49,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +19: [2023-05-13 09:38:49,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:49,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:49,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:49,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:49,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 6: [2023-05-13 09:38:49,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:49,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 4: [2023-05-13 09:38:49,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 4: [2023-05-13 09:38:49,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 4: [2023-05-13 09:38:49,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:49,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +16: [2023-05-13 09:38:49,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +29: [2023-05-13 09:38:49,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:49,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +29: [2023-05-13 09:38:49,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +29: [2023-05-13 09:38:49,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +30: [2023-05-13 09:38:49,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:49,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +25: [2023-05-13 09:38:49,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +23: [2023-05-13 09:38:49,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +15: [2023-05-13 09:38:49,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:49,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:49,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +25: [2023-05-13 09:38:49,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +30: [2023-05-13 09:38:49,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:49,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +29: [2023-05-13 09:38:49,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:49,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +25: [2023-05-13 09:38:49,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:49,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +15: [2023-05-13 09:38:49,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:49,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:49,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:49,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:49,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:49,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:49,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:49,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 9: [2023-05-13 09:38:49,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 1: [2023-05-13 09:38:49,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +22: [2023-05-13 09:38:49,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:49,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:49,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:49,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +31: [2023-05-13 09:38:49,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:49,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:49,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:49,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:49,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:49,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +22: [2023-05-13 09:38:49,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:49,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +11: [2023-05-13 09:38:49,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:49,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:49,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:49,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +28: [2023-05-13 09:38:49,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +28: [2023-05-13 09:38:49,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. +28: [2023-05-13 09:38:49,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_08-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +17: [2023-05-13 09:38:49,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 9: [2023-05-13 09:38:49,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:49,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:49,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:49,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:49,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +16: [2023-05-13 09:38:49,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:49,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:49,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +10: [2023-05-13 09:38:49,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +10: [2023-05-13 09:38:49,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:49,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:49,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 5: [2023-05-13 09:38:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 5: [2023-05-13 09:38:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:49,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:49,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:49,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:49,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:49,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:49,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:49,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +15: [2023-05-13 09:38:49,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:49,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +31: [2023-05-13 09:38:49,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:49,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:49,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:49,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:49,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:49,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +28: [2023-05-13 09:38:49,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +23: [2023-05-13 09:38:49,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:49,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt... +17: [2023-05-13 09:38:49,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:49,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:49,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:49,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +21: [2023-05-13 09:38:49,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:49,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +13: [2023-05-13 09:38:49,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +21: [2023-05-13 09:38:49,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:49,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:49,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +23: [2023-05-13 09:38:49,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:49,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:49,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:49,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:49,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:49,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:49,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:49,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +26: [2023-05-13 09:38:49,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:49,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +26: [2023-05-13 09:38:49,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:49,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:49,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +26: [2023-05-13 09:38:49,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:49,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:49,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 2: [2023-05-13 09:38:49,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:49,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +26: [2023-05-13 09:38:49,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +11: [2023-05-13 09:38:49,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +13: [2023-05-13 09:38:49,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:49,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:49,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:49,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:49,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:49,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:49,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +30: [2023-05-13 09:38:49,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 2: [2023-05-13 09:38:49,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:49,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +11: [2023-05-13 09:38:49,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:49,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:49,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:49,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +30: [2023-05-13 09:38:49,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:49,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:49,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:49,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +15: [2023-05-13 09:38:49,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 8: [2023-05-13 09:38:49,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:49,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +16: [2023-05-13 09:38:49,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:49,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +23: [2023-05-13 09:38:49,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:49,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:49,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:49,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:49,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:49,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +16: [2023-05-13 09:38:49,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:49,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:49,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:49,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +10: [2023-05-13 09:38:49,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +23: [2023-05-13 09:38:49,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:49,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:49,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:49,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +31: [2023-05-13 09:38:49,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:49,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:49,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:49,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:49,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:49,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 9: [2023-05-13 09:38:49,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:49,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:49,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:49,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:49,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +23: [2023-05-13 09:38:49,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:49,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +28: [2023-05-13 09:38:49,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +28: [2023-05-13 09:38:49,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +31: [2023-05-13 09:38:49,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:49,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 8: [2023-05-13 09:38:49,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:49,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +28: [2023-05-13 09:38:49,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +23: [2023-05-13 09:38:49,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 9: [2023-05-13 09:38:49,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_00-model_states.pt. +17: [2023-05-13 09:38:49,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +14: [2023-05-13 09:38:49,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:49,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +14: [2023-05-13 09:38:49,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +17: [2023-05-13 09:38:49,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 9: [2023-05-13 09:38:49,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +17: [2023-05-13 09:38:49,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +17: [2023-05-13 09:38:49,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:49,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt... +17: [2023-05-13 09:38:49,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 9: [2023-05-13 09:38:49,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 0: [2023-05-13 09:38:49,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 0: [2023-05-13 09:38:49,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 0: [2023-05-13 09:38:49,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +14: [2023-05-13 09:38:49,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:49,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:49,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:49,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:49,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:49,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:49,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:49,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:49,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:49,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:49,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:49,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:49,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:49,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:49,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:49,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:49,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:49,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:49,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:49,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:49,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +24: [2023-05-13 09:38:49,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +24: [2023-05-13 09:38:49,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +24: [2023-05-13 09:38:49,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +14: [2023-05-13 09:38:49,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 0: [2023-05-13 09:38:49,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:49,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 0: [2023-05-13 09:38:49,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 2: [2023-05-13 09:38:49,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:49,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:49,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:49,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:49,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:49,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:49,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:49,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:49,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:49,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:49,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:49,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:49,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:49,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:49,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:49,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:49,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:49,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:49,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:49,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:49,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:49,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +18: [2023-05-13 09:38:49,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:49,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:49,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +15: [2023-05-13 09:38:49,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:49,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:49,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:49,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:49,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:49,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:49,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:49,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:49,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:49,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:49,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:49,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:49,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:49,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:49,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:49,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:49,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +24: [2023-05-13 09:38:49,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +24: [2023-05-13 09:38:49,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 0: [2023-05-13 09:38:49,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +24: [2023-05-13 09:38:49,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 2: [2023-05-13 09:38:49,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:49,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:49,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:49,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:49,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:49,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:49,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:49,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:49,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:49,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:49,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:49,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +15: [2023-05-13 09:38:49,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:49,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:49,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:49,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:49,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:49,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:49,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:49,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:49,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:49,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +27: [2023-05-13 09:38:49,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:49,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:49,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:49,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:49,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:49,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:49,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:49,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:49,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:49,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:49,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:49,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:49,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +18: [2023-05-13 09:38:49,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:49,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:49,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:49,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:49,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:49,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:49,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:49,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:49,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:49,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:49,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:49,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:49,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:49,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:49,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:49,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:49,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:49,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:49,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:49,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:49,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:49,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:49,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:49,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:49,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:49,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:49,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:49,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:49,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:49,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:49,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:49,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:49,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +12: [2023-05-13 09:38:49,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +21: [2023-05-13 09:38:49,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +15: [2023-05-13 09:38:49,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +15: [2023-05-13 09:38:49,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +12: [2023-05-13 09:38:49,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:49,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:49,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:49,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:49,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:49,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:49,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +12: [2023-05-13 09:38:49,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +20: [2023-05-13 09:38:49,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 6: [2023-05-13 09:38:49,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 6: [2023-05-13 09:38:49,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:49,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +20: [2023-05-13 09:38:49,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:49,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:49,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:49,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:49,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:49,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:49,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:49,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:49,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +12: [2023-05-13 09:38:49,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +20: [2023-05-13 09:38:49,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:49,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:49,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:49,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:49,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:49,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:49,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:49,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:49,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:49,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:49,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:49,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:49,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:49,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:49,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:49,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:49,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:49,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:49,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:49,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +12: [2023-05-13 09:38:49,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:49,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:49,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:49,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +18: [2023-05-13 09:38:49,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +21: [2023-05-13 09:38:49,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:49,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:49,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:49,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +26: [2023-05-13 09:38:49,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:49,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:49,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +18: [2023-05-13 09:38:49,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:49,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:49,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:49,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:49,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +20: [2023-05-13 09:38:49,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:49,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:49,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +18: [2023-05-13 09:38:49,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +26: [2023-05-13 09:38:49,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:49,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:49,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +19: [2023-05-13 09:38:49,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:49,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:49,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:49,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:49,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +31: [2023-05-13 09:38:49,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:49,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:49,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:49,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:49,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:49,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:49,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:49,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +26: [2023-05-13 09:38:49,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:49,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +26: [2023-05-13 09:38:49,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:49,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +26: [2023-05-13 09:38:49,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:49,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:49,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:49,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +20: [2023-05-13 09:38:49,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:49,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +22: [2023-05-13 09:38:49,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +22: [2023-05-13 09:38:49,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +23: [2023-05-13 09:38:49,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:49,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +23: [2023-05-13 09:38:49,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:49,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:49,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:49,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:49,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:49,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:49,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:49,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:49,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:49,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:49,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:49,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:49,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:49,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:49,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:49,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:49,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:49,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:49,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +19: [2023-05-13 09:38:49,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:49,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:49,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:49,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:49,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:49,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:49,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:49,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:49,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:49,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:49,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:49,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:49,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:49,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:49,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:49,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:49,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +22: [2023-05-13 09:38:49,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +22: [2023-05-13 09:38:49,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:49,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:49,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +21: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:49,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:49,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 6: [2023-05-13 09:38:49,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:49,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:49,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:49,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:49,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +25: [2023-05-13 09:38:49,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:49,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:49,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:49,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:49,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 1: [2023-05-13 09:38:49,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +19: [2023-05-13 09:38:49,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +19: [2023-05-13 09:38:49,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +20: [2023-05-13 09:38:49,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +20: [2023-05-13 09:38:49,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +20: [2023-05-13 09:38:49,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:49,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +20: [2023-05-13 09:38:49,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +23: [2023-05-13 09:38:49,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:49,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:49,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:49,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:49,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:49,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:49,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +26: [2023-05-13 09:38:49,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +26: [2023-05-13 09:38:49,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:49,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:49,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:49,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:49,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:49,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +26: [2023-05-13 09:38:49,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +19: [2023-05-13 09:38:49,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +26: [2023-05-13 09:38:49,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +19: [2023-05-13 09:38:49,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +22: [2023-05-13 09:38:49,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:49,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:49,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:49,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:49,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:49,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +25: [2023-05-13 09:38:49,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:49,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 6: [2023-05-13 09:38:49,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:49,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:49,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:49,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:49,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:49,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:49,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:49,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +17: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:49,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:49,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:49,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:49,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:49,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:49,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:49,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:49,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:49,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 7: [2023-05-13 09:38:49,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +17: [2023-05-13 09:38:49,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +17: [2023-05-13 09:38:49,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:49,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 0: [2023-05-13 09:38:49,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 0: [2023-05-13 09:38:49,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 0: [2023-05-13 09:38:49,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:49,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:49,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:49,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +17: [2023-05-13 09:38:49,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:49,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 6: [2023-05-13 09:38:49,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:49,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:49,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:49,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 1: [2023-05-13 09:38:49,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +15: [2023-05-13 09:38:49,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +24: [2023-05-13 09:38:49,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 8: [2023-05-13 09:38:49,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 8: [2023-05-13 09:38:49,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:49,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:49,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:49,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:49,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 4: [2023-05-13 09:38:49,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 4: [2023-05-13 09:38:49,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:49,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +23: [2023-05-13 09:38:49,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 4: [2023-05-13 09:38:49,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +23: [2023-05-13 09:38:49,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:49,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +25: [2023-05-13 09:38:49,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:49,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +20: [2023-05-13 09:38:49,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:49,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:49,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 3: [2023-05-13 09:38:49,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 3: [2023-05-13 09:38:49,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:49,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:49,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:49,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +20: [2023-05-13 09:38:49,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +20: [2023-05-13 09:38:49,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +20: [2023-05-13 09:38:49,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:49,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 8: [2023-05-13 09:38:49,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:49,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:49,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:49,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:49,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:49,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:49,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:49,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:49,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:49,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +13: [2023-05-13 09:38:49,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:49,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +23: [2023-05-13 09:38:49,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +25: [2023-05-13 09:38:49,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:49,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:49,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +17: [2023-05-13 09:38:49,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:49,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +17: [2023-05-13 09:38:49,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 8: [2023-05-13 09:38:49,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 8: [2023-05-13 09:38:49,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 8: [2023-05-13 09:38:49,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:49,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:49,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:49,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:49,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +12: [2023-05-13 09:38:49,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +12: [2023-05-13 09:38:49,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +12: [2023-05-13 09:38:49,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +12: [2023-05-13 09:38:49,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:49,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:49,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:49,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:49,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:49,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:49,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:49,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:49,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:49,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:49,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:49,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:49,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:49,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:49,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:49,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:49,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:49,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +31: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 8: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +15: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +15: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +27: [2023-05-13 09:38:49,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:49,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:49,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +15: [2023-05-13 09:38:49,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:49,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:49,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +27: [2023-05-13 09:38:49,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +23: [2023-05-13 09:38:49,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +31: [2023-05-13 09:38:49,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:49,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +23: [2023-05-13 09:38:49,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:49,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:49,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:49,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +23: [2023-05-13 09:38:49,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +13: [2023-05-13 09:38:49,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:49,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +29: [2023-05-13 09:38:49,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +29: [2023-05-13 09:38:49,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:49,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:49,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +13: [2023-05-13 09:38:49,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:49,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +21: [2023-05-13 09:38:49,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:49,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +13: [2023-05-13 09:38:49,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:49,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +21: [2023-05-13 09:38:49,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +14: [2023-05-13 09:38:49,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 9: [2023-05-13 09:38:49,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 9: [2023-05-13 09:38:49,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:49,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +12: [2023-05-13 09:38:49,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:49,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:49,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +11: [2023-05-13 09:38:49,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 2: [2023-05-13 09:38:49,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +15: [2023-05-13 09:38:49,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:49,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:49,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:49,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:49,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:49,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:49,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:49,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:49,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:49,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:49,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:49,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:49,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +21: [2023-05-13 09:38:49,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +31: [2023-05-13 09:38:49,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +21: [2023-05-13 09:38:49,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:49,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:49,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:49,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:49,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:49,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:49,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +30: [2023-05-13 09:38:49,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:49,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:49,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:49,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:49,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:49,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +28: [2023-05-13 09:38:49,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:49,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +10: [2023-05-13 09:38:49,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +27: [2023-05-13 09:38:49,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:49,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:49,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:49,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:49,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:49,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:49,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:49,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:49,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:49,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:49,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +27: [2023-05-13 09:38:49,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:49,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:49,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:49,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +31: [2023-05-13 09:38:49,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:49,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:49,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:49,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:49,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:49,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:49,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:49,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +27: [2023-05-13 09:38:49,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:49,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:49,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +15: [2023-05-13 09:38:49,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +27: [2023-05-13 09:38:49,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:49,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:49,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +14: [2023-05-13 09:38:49,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:49,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:49,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:49,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:49,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:49,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:49,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:49,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +10: [2023-05-13 09:38:49,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:49,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:49,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +28: [2023-05-13 09:38:49,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +29: [2023-05-13 09:38:49,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +29: [2023-05-13 09:38:49,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +27: [2023-05-13 09:38:49,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +10: [2023-05-13 09:38:49,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +14: [2023-05-13 09:38:49,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +30: [2023-05-13 09:38:49,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:49,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:49,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +30: [2023-05-13 09:38:49,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:49,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +16: [2023-05-13 09:38:49,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +16: [2023-05-13 09:38:49,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +16: [2023-05-13 09:38:49,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_09-model_01-model_states.pt. +29: [2023-05-13 09:38:49,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +29: [2023-05-13 09:38:49,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +10: [2023-05-13 09:38:49,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:49,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:49,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +21: [2023-05-13 09:38:49,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +11: [2023-05-13 09:38:49,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +14: [2023-05-13 09:38:49,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +21: [2023-05-13 09:38:49,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:49,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +11: [2023-05-13 09:38:49,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +12: [2023-05-13 09:38:49,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +14: [2023-05-13 09:38:49,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +21: [2023-05-13 09:38:49,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +21: [2023-05-13 09:38:49,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:49,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +14: [2023-05-13 09:38:49,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:49,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:49,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:49,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +28: [2023-05-13 09:38:49,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:49,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +12: [2023-05-13 09:38:49,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +12: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +12: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +19: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 6: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 6: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:49,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:49,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 9: [2023-05-13 09:38:49,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +12: [2023-05-13 09:38:49,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:49,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:49,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 0: [2023-05-13 09:38:49,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:49,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:49,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:49,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:49,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:49,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:49,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:49,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 0: [2023-05-13 09:38:49,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +10: [2023-05-13 09:38:49,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:49,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:49,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... +16: [2023-05-13 09:38:49,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:49,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:49,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:49,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:49,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:49,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:49,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:49,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +14: [2023-05-13 09:38:49,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:49,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:49,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +22: [2023-05-13 09:38:49,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:49,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +19: [2023-05-13 09:38:49,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +27: [2023-05-13 09:38:49,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:49,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:49,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:49,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:49,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:49,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:49,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +10: [2023-05-13 09:38:49,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:49,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +10: [2023-05-13 09:38:49,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:49,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:49,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +30: [2023-05-13 09:38:49,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +28: [2023-05-13 09:38:49,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +10: [2023-05-13 09:38:49,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +10: [2023-05-13 09:38:49,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +10: [2023-05-13 09:38:49,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +28: [2023-05-13 09:38:49,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +22: [2023-05-13 09:38:49,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +22: [2023-05-13 09:38:49,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:49,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:49,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:49,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:49,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +22: [2023-05-13 09:38:49,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:49,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +30: [2023-05-13 09:38:49,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +27: [2023-05-13 09:38:49,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:49,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:49,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:49,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:49,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:49,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:49,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +12: [2023-05-13 09:38:49,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:49,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +16: [2023-05-13 09:38:49,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_00-model_states.pt. +11: [2023-05-13 09:38:49,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:49,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:49,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 5: [2023-05-13 09:38:49,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:49,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +16: [2023-05-13 09:38:49,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... + 2: [2023-05-13 09:38:49,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt... +19: [2023-05-13 09:38:49,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:49,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:49,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:49,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:49,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:49,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:49,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:49,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:49,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:49,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:49,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:49,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:49,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +12: [2023-05-13 09:38:49,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:49,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 6: [2023-05-13 09:38:49,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:49,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:49,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:49,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:49,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:49,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:49,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:49,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:49,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 6: [2023-05-13 09:38:49,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:49,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +12: [2023-05-13 09:38:49,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:49,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:49,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:49,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:49,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 0: [2023-05-13 09:38:49,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:49,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +24: [2023-05-13 09:38:49,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +24: [2023-05-13 09:38:49,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +24: [2023-05-13 09:38:49,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 0: [2023-05-13 09:38:49,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:49,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:49,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:49,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:49,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:49,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:49,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +18: [2023-05-13 09:38:49,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +18: [2023-05-13 09:38:49,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:49,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:49,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:49,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:49,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:49,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:49,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:49,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 0: [2023-05-13 09:38:49,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:49,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:49,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:49,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:49,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:49,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:49,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:49,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:49,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:49,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:49,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +13: [2023-05-13 09:38:49,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:49,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:49,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:49,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:49,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:49,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:49,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:49,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:49,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:49,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:49,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:49,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:49,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:49,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:49,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +16: [2023-05-13 09:38:49,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:49,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:49,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +22: [2023-05-13 09:38:49,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:49,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +22: [2023-05-13 09:38:49,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +22: [2023-05-13 09:38:49,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 8: [2023-05-13 09:38:49,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:49,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:49,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:49,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:49,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +25: [2023-05-13 09:38:49,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +25: [2023-05-13 09:38:49,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +25: [2023-05-13 09:38:49,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 1: [2023-05-13 09:38:49,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +25: [2023-05-13 09:38:49,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 7: [2023-05-13 09:38:49,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:49,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +17: [2023-05-13 09:38:49,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:49,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:49,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:49,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +24: [2023-05-13 09:38:49,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:49,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +17: [2023-05-13 09:38:49,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:49,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:49,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:49,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:49,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 5: [2023-05-13 09:38:49,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +24: [2023-05-13 09:38:49,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:49,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:49,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:49,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:49,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:49,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:49,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:49,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +23: [2023-05-13 09:38:49,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:49,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +23: [2023-05-13 09:38:49,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:49,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:49,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:49,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:49,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +19: [2023-05-13 09:38:49,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:49,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:49,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 4: [2023-05-13 09:38:49,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +19: [2023-05-13 09:38:49,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:49,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:49,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:49,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +16: [2023-05-13 09:38:49,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:49,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:49,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:49,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:49,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:49,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:49,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:49,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:49,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:49,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:49,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:49,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:49,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +13: [2023-05-13 09:38:49,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +13: [2023-05-13 09:38:49,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +13: [2023-05-13 09:38:49,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +23: [2023-05-13 09:38:49,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:49,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 5: [2023-05-13 09:38:49,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:49,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:49,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:49,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:49,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:49,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:49,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:49,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +20: [2023-05-13 09:38:49,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +20: [2023-05-13 09:38:49,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:49,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +16: [2023-05-13 09:38:49,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +16: [2023-05-13 09:38:49,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:49,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:49,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +17: [2023-05-13 09:38:49,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +17: [2023-05-13 09:38:49,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +17: [2023-05-13 09:38:49,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +17: [2023-05-13 09:38:49,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +17: [2023-05-13 09:38:49,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:49,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +19: [2023-05-13 09:38:49,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:49,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +20: [2023-05-13 09:38:49,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:49,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:49,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +15: [2023-05-13 09:38:49,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:49,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:49,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +19: [2023-05-13 09:38:49,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:49,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:49,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:49,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 8: [2023-05-13 09:38:49,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +17: [2023-05-13 09:38:49,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:49,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:49,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:49,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +29: [2023-05-13 09:38:49,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +17: [2023-05-13 09:38:49,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:49,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:49,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:49,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:49,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:49,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:49,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:49,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:49,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:49,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:49,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:49,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:49,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:49,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:49,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:49,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:49,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +19: [2023-05-13 09:38:49,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:49,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:49,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:49,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:49,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:49,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:49,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:49,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:49,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:49,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:49,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 4: [2023-05-13 09:38:49,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 4: [2023-05-13 09:38:49,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:49,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:49,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +14: [2023-05-13 09:38:49,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:49,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:49,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:49,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 5: [2023-05-13 09:38:49,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 7: [2023-05-13 09:38:49,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:49,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +14: [2023-05-13 09:38:49,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:49,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +29: [2023-05-13 09:38:49,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:49,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +27: [2023-05-13 09:38:49,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +27: [2023-05-13 09:38:49,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 7: [2023-05-13 09:38:49,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:49,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 1: [2023-05-13 09:38:49,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:49,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 1: [2023-05-13 09:38:49,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:49,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:49,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:49,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +17: [2023-05-13 09:38:49,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:49,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:49,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:49,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:49,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +21: [2023-05-13 09:38:49,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +21: [2023-05-13 09:38:49,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +21: [2023-05-13 09:38:49,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +17: [2023-05-13 09:38:49,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:49,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:49,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +20: [2023-05-13 09:38:49,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:49,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:49,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:49,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:49,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:49,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 2: [2023-05-13 09:38:49,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:49,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:49,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +26: [2023-05-13 09:38:49,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:49,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:49,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +26: [2023-05-13 09:38:49,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:49,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +11: [2023-05-13 09:38:49,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +11: [2023-05-13 09:38:49,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +11: [2023-05-13 09:38:49,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:49,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:49,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +28: [2023-05-13 09:38:49,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +28: [2023-05-13 09:38:49,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 9: [2023-05-13 09:38:49,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 9: [2023-05-13 09:38:49,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +28: [2023-05-13 09:38:49,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +26: [2023-05-13 09:38:49,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +20: [2023-05-13 09:38:49,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 2: [2023-05-13 09:38:49,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. + 3: [2023-05-13 09:38:49,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +26: [2023-05-13 09:38:49,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:49,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:49,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:49,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:49,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:49,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:49,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 2: [2023-05-13 09:38:49,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:49,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:49,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:49,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +14: [2023-05-13 09:38:49,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:49,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:49,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:49,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:49,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:49,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +27: [2023-05-13 09:38:49,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:49,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +13: [2023-05-13 09:38:49,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +13: [2023-05-13 09:38:49,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +25: [2023-05-13 09:38:49,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:49,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:49,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:49,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:49,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +20: [2023-05-13 09:38:49,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +27: [2023-05-13 09:38:49,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:49,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:49,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:49,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:49,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 3: [2023-05-13 09:38:49,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +25: [2023-05-13 09:38:49,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:49,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:49,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:49,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:49,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:49,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:49,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:49,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:49,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:49,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:49,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:49,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:49,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:49,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:49,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:49,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:49,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:49,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +13: [2023-05-13 09:38:49,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:49,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:49,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:49,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:49,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:49,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:49,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +28: [2023-05-13 09:38:49,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +11: [2023-05-13 09:38:49,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +15: [2023-05-13 09:38:49,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +13: [2023-05-13 09:38:49,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:49,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:49,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... + 9: [2023-05-13 09:38:49,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:49,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +15: [2023-05-13 09:38:49,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:49,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:49,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:49,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +20: [2023-05-13 09:38:49,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:49,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:49,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +20: [2023-05-13 09:38:49,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:49,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +15: [2023-05-13 09:38:49,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:49,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:49,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:49,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 7: [2023-05-13 09:38:49,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:49,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:49,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:49,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:49,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:49,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:49,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:49,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:49,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:49,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:49,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:49,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +15: [2023-05-13 09:38:49,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:50,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:50,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:50,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:50,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +16: [2023-05-13 09:38:50,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +16: [2023-05-13 09:38:50,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +16: [2023-05-13 09:38:50,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +26: [2023-05-13 09:38:50,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:50,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:50,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:50,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:50,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:50,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:50,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +31: [2023-05-13 09:38:50,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:50,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:50,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +15: [2023-05-13 09:38:50,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:50,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +31: [2023-05-13 09:38:50,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:50,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:50,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +25: [2023-05-13 09:38:50,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:50,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:50,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:50,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +26: [2023-05-13 09:38:50,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:50,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +28: [2023-05-13 09:38:50,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +11: [2023-05-13 09:38:50,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +12: [2023-05-13 09:38:50,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +12: [2023-05-13 09:38:50,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +12: [2023-05-13 09:38:50,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +26: [2023-05-13 09:38:50,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +11: [2023-05-13 09:38:50,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +26: [2023-05-13 09:38:50,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +11: [2023-05-13 09:38:50,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:50,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +15: [2023-05-13 09:38:50,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:50,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:50,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +11: [2023-05-13 09:38:50,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:50,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +18: [2023-05-13 09:38:50,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:50,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:50,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:50,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:50,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:50,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:50,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +26: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 0: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +23: [2023-05-13 09:38:50,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +25: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +18: [2023-05-13 09:38:50,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:50,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +25: [2023-05-13 09:38:50,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:50,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:50,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:50,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:50,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +18: [2023-05-13 09:38:50,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +18: [2023-05-13 09:38:50,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +15: [2023-05-13 09:38:50,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:50,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:50,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:50,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:50,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:50,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +28: [2023-05-13 09:38:50,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:50,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:50,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:50,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:50,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:50,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:50,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:50,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:50,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:50,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +28: [2023-05-13 09:38:50,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 9: [2023-05-13 09:38:50,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:50,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:50,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +21: [2023-05-13 09:38:50,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:50,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:50,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +10: [2023-05-13 09:38:50,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +31: [2023-05-13 09:38:50,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:50,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:50,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:50,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:50,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:50,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +30: [2023-05-13 09:38:50,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_10-model_01-model_states.pt. +25: [2023-05-13 09:38:50,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:50,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:50,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:50,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:50,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:50,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:50,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 9: [2023-05-13 09:38:50,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +21: [2023-05-13 09:38:50,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:50,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:50,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:50,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:50,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:50,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +27: [2023-05-13 09:38:50,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:50,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 6: [2023-05-13 09:38:50,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 6: [2023-05-13 09:38:50,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 6: [2023-05-13 09:38:50,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 0: [2023-05-13 09:38:50,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:50,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:50,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:50,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:50,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:50,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:50,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:50,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:50,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +27: [2023-05-13 09:38:50,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +27: [2023-05-13 09:38:50,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:50,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 3: [2023-05-13 09:38:50,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:50,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +27: [2023-05-13 09:38:50,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +27: [2023-05-13 09:38:50,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:50,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:50,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:50,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:50,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:50,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:50,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:50,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +23: [2023-05-13 09:38:50,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:50,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:50,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:50,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:50,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +10: [2023-05-13 09:38:50,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +18: [2023-05-13 09:38:50,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:50,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:50,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:50,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:50,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:50,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +30: [2023-05-13 09:38:50,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt... +16: [2023-05-13 09:38:50,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +29: [2023-05-13 09:38:50,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:50,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:50,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:50,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:50,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:50,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:50,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +16: [2023-05-13 09:38:50,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +18: [2023-05-13 09:38:50,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:50,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:50,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:50,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:50,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +16: [2023-05-13 09:38:50,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +21: [2023-05-13 09:38:50,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:50,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:50,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:50,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:50,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +23: [2023-05-13 09:38:50,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:50,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:50,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +27: [2023-05-13 09:38:50,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:50,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:50,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:50,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:50,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:50,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:50,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +18: [2023-05-13 09:38:50,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +27: [2023-05-13 09:38:50,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:50,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +19: [2023-05-13 09:38:50,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:50,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:50,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:50,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:50,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:50,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:50,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +10: [2023-05-13 09:38:50,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +24: [2023-05-13 09:38:50,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:50,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:50,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +10: [2023-05-13 09:38:50,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +23: [2023-05-13 09:38:50,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:50,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:50,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:50,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:50,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:50,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. +30: [2023-05-13 09:38:50,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +27: [2023-05-13 09:38:50,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:50,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +30: [2023-05-13 09:38:50,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +30: [2023-05-13 09:38:50,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +24: [2023-05-13 09:38:50,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:50,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt... +29: [2023-05-13 09:38:50,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:50,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:50,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +19: [2023-05-13 09:38:50,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +19: [2023-05-13 09:38:50,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:50,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +19: [2023-05-13 09:38:50,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +31: [2023-05-13 09:38:50,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:50,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:50,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:50,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:50,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:50,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:50,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:50,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:50,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +31: [2023-05-13 09:38:50,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +31: [2023-05-13 09:38:50,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +31: [2023-05-13 09:38:50,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +31: [2023-05-13 09:38:50,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:50,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:50,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:50,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:50,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +22: [2023-05-13 09:38:50,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +31: [2023-05-13 09:38:50,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:50,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:50,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:50,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:50,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:50,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:50,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 1: [2023-05-13 09:38:50,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +24: [2023-05-13 09:38:50,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +24: [2023-05-13 09:38:50,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +24: [2023-05-13 09:38:50,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +24: [2023-05-13 09:38:50,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 1: [2023-05-13 09:38:50,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 1: [2023-05-13 09:38:50,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +31: [2023-05-13 09:38:50,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:50,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +31: [2023-05-13 09:38:50,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:50,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:50,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:50,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +24: [2023-05-13 09:38:50,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +31: [2023-05-13 09:38:50,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:50,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +25: [2023-05-13 09:38:50,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:50,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:50,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +14: [2023-05-13 09:38:50,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:50,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:50,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:50,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:50,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +14: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +14: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:50,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:50,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:50,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:50,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:50,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:50,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:50,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:50,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 2: [2023-05-13 09:38:50,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +22: [2023-05-13 09:38:50,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:50,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:50,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:50,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:50,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:50,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:50,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:50,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +12: [2023-05-13 09:38:50,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +12: [2023-05-13 09:38:50,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 9: [2023-05-13 09:38:50,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:50,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:50,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +12: [2023-05-13 09:38:50,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:50,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:50,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:50,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:50,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 1: [2023-05-13 09:38:50,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 1: [2023-05-13 09:38:50,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:50,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:50,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:50,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +24: [2023-05-13 09:38:50,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:50,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +24: [2023-05-13 09:38:50,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +24: [2023-05-13 09:38:50,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +22: [2023-05-13 09:38:50,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:50,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:50,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 2: [2023-05-13 09:38:50,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 2: [2023-05-13 09:38:50,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +11: [2023-05-13 09:38:50,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:50,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:50,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +11: [2023-05-13 09:38:50,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:50,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:50,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:50,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:50,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:50,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:50,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:50,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:50,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:50,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:50,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +22: [2023-05-13 09:38:50,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:50,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:50,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:50,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:50,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:50,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:50,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:50,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +25: [2023-05-13 09:38:50,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +25: [2023-05-13 09:38:50,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:50,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:50,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:50,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +25: [2023-05-13 09:38:50,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:50,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +12: [2023-05-13 09:38:50,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +14: [2023-05-13 09:38:50,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:50,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:50,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:50,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:50,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:50,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 9: [2023-05-13 09:38:50,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:50,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:50,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:50,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:50,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:50,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:50,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:50,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:50,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:50,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +12: [2023-05-13 09:38:50,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:50,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:50,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:50,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 4: [2023-05-13 09:38:50,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:50,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:50,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:50,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:50,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:50,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 7: [2023-05-13 09:38:50,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 7: [2023-05-13 09:38:50,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 7: [2023-05-13 09:38:50,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:50,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:50,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +20: [2023-05-13 09:38:50,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:50,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:50,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:50,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 4: [2023-05-13 09:38:50,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 4: [2023-05-13 09:38:50,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 4: [2023-05-13 09:38:50,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:50,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:50,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:50,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:50,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:50,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +11: [2023-05-13 09:38:50,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 2: [2023-05-13 09:38:50,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:50,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:50,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +28: [2023-05-13 09:38:50,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:50,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:50,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:50,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 7: [2023-05-13 09:38:50,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:50,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:50,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:50,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:50,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:50,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:50,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:50,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:50,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:50,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:50,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:50,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:50,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +13: [2023-05-13 09:38:50,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:50,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:50,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:50,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:50,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:50,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:50,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +11: [2023-05-13 09:38:50,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +11: [2023-05-13 09:38:50,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +11: [2023-05-13 09:38:50,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +29: [2023-05-13 09:38:50,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +29: [2023-05-13 09:38:50,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +29: [2023-05-13 09:38:50,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +29: [2023-05-13 09:38:50,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:50,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:50,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:50,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:50,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:50,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:50,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:50,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:50,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:50,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:50,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:50,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:50,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:50,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:50,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:50,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:50,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:50,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:50,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:50,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:50,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:50,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +28: [2023-05-13 09:38:50,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:50,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:50,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:50,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:50,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:50,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:50,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +13: [2023-05-13 09:38:50,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:50,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:50,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:50,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:50,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:50,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:50,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:50,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:50,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:50,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +21: [2023-05-13 09:38:50,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +21: [2023-05-13 09:38:50,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +21: [2023-05-13 09:38:50,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:50,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 3: [2023-05-13 09:38:50,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 4: [2023-05-13 09:38:50,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:50,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:50,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:50,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +23: [2023-05-13 09:38:50,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:50,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:50,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:50,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:50,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:50,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:50,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +23: [2023-05-13 09:38:50,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:50,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 9: [2023-05-13 09:38:50,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +13: [2023-05-13 09:38:50,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +13: [2023-05-13 09:38:50,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 4: [2023-05-13 09:38:50,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 0: [2023-05-13 09:38:50,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:50,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +13: [2023-05-13 09:38:50,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:50,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +16: [2023-05-13 09:38:50,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:50,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:50,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. + 5: [2023-05-13 09:38:50,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:50,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +29: [2023-05-13 09:38:50,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +10: [2023-05-13 09:38:50,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +10: [2023-05-13 09:38:50,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +28: [2023-05-13 09:38:50,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:50,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +29: [2023-05-13 09:38:50,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:50,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:50,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:50,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:50,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:50,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:50,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:50,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:50,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:50,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:50,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:50,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +26: [2023-05-13 09:38:50,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:50,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +13: [2023-05-13 09:38:50,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +13: [2023-05-13 09:38:50,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:50,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:50,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:50,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:50,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +29: [2023-05-13 09:38:50,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:50,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:50,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:50,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:50,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:50,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +23: [2023-05-13 09:38:50,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +29: [2023-05-13 09:38:50,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 3: [2023-05-13 09:38:50,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:50,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +29: [2023-05-13 09:38:50,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +29: [2023-05-13 09:38:50,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +11: [2023-05-13 09:38:50,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 9: [2023-05-13 09:38:50,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:50,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +11: [2023-05-13 09:38:50,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:50,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:50,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:50,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:50,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:50,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +11: [2023-05-13 09:38:50,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +11: [2023-05-13 09:38:50,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:50,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +16: [2023-05-13 09:38:50,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:50,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +10: [2023-05-13 09:38:50,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +17: [2023-05-13 09:38:50,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:50,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:50,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:50,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +28: [2023-05-13 09:38:50,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +15: [2023-05-13 09:38:50,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:50,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:50,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +26: [2023-05-13 09:38:50,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:50,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:50,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:50,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:50,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:50,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:50,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:50,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:50,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 5: [2023-05-13 09:38:50,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:50,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:50,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:50,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:50,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:50,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:50,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:50,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:50,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:50,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +18: [2023-05-13 09:38:50,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:50,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:50,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:50,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:50,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:50,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:50,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:50,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:50,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:50,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +19: [2023-05-13 09:38:50,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:50,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:50,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +28: [2023-05-13 09:38:50,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:50,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:50,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:50,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 3: [2023-05-13 09:38:50,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 3: [2023-05-13 09:38:50,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:50,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +26: [2023-05-13 09:38:50,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:50,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:50,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:50,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:50,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:50,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +30: [2023-05-13 09:38:50,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_11-model_01-model_states.pt. +19: [2023-05-13 09:38:50,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:50,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:50,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:50,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:50,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:50,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:50,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:50,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +21: [2023-05-13 09:38:50,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +19: [2023-05-13 09:38:50,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +16: [2023-05-13 09:38:50,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +16: [2023-05-13 09:38:50,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:50,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:50,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:50,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:50,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 5: [2023-05-13 09:38:50,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:50,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +21: [2023-05-13 09:38:50,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 5: [2023-05-13 09:38:50,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +17: [2023-05-13 09:38:50,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:50,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +23: [2023-05-13 09:38:50,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:50,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:50,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:50,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:50,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:50,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:50,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +26: [2023-05-13 09:38:50,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +10: [2023-05-13 09:38:50,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +17: [2023-05-13 09:38:50,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +26: [2023-05-13 09:38:50,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:50,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:50,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:50,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:50,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:50,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +30: [2023-05-13 09:38:50,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +18: [2023-05-13 09:38:50,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:50,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:50,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt... +21: [2023-05-13 09:38:50,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:50,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:50,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:50,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:50,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:50,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:50,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:50,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:50,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:50,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +17: [2023-05-13 09:38:50,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +17: [2023-05-13 09:38:50,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +26: [2023-05-13 09:38:50,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:50,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:50,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 7: [2023-05-13 09:38:50,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:50,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 7: [2023-05-13 09:38:50,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +18: [2023-05-13 09:38:50,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 7: [2023-05-13 09:38:50,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +21: [2023-05-13 09:38:50,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:50,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +18: [2023-05-13 09:38:50,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:50,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:50,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:50,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:50,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 6: [2023-05-13 09:38:50,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 6: [2023-05-13 09:38:50,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:50,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:50,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 6: [2023-05-13 09:38:50,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +27: [2023-05-13 09:38:50,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:50,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +27: [2023-05-13 09:38:50,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +27: [2023-05-13 09:38:50,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +15: [2023-05-13 09:38:50,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +27: [2023-05-13 09:38:50,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:50,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +23: [2023-05-13 09:38:50,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:50,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:50,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:50,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_00-model_states.pt. +30: [2023-05-13 09:38:50,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:50,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... +30: [2023-05-13 09:38:50,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:50,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:50,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:50,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:50,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:50,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:50,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:50,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 7: [2023-05-13 09:38:50,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:50,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:50,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +12: [2023-05-13 09:38:50,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:50,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:50,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:50,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:50,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:50,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:50,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:50,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +24: [2023-05-13 09:38:50,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +24: [2023-05-13 09:38:50,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 7: [2023-05-13 09:38:50,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:50,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +27: [2023-05-13 09:38:50,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:50,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:50,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:50,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:50,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:50,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:50,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:50,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:50,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:50,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:50,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +31: [2023-05-13 09:38:50,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +31: [2023-05-13 09:38:50,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +15: [2023-05-13 09:38:50,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:50,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:50,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +27: [2023-05-13 09:38:50,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +15: [2023-05-13 09:38:50,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +27: [2023-05-13 09:38:50,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +27: [2023-05-13 09:38:50,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:50,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:50,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +24: [2023-05-13 09:38:50,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:50,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +14: [2023-05-13 09:38:50,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +24: [2023-05-13 09:38:50,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:50,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:50,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +14: [2023-05-13 09:38:50,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +29: [2023-05-13 09:38:50,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:50,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:50,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:50,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:50,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:50,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:50,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:50,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:50,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:50,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:50,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:50,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:50,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:50,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:50,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +31: [2023-05-13 09:38:50,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +24: [2023-05-13 09:38:50,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +27: [2023-05-13 09:38:50,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:50,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:50,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:50,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:50,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:50,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:50,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:50,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +29: [2023-05-13 09:38:50,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:50,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:50,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 1: [2023-05-13 09:38:50,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 1: [2023-05-13 09:38:50,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 1: [2023-05-13 09:38:50,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +31: [2023-05-13 09:38:50,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:50,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:50,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:50,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:50,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:50,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:50,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:50,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:50,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +24: [2023-05-13 09:38:50,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +24: [2023-05-13 09:38:50,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +24: [2023-05-13 09:38:50,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:50,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +31: [2023-05-13 09:38:50,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:50,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +24: [2023-05-13 09:38:50,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +24: [2023-05-13 09:38:50,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:50,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:50,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:50,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 0: [2023-05-13 09:38:50,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 0: [2023-05-13 09:38:50,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 0: [2023-05-13 09:38:50,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 0: [2023-05-13 09:38:50,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:50,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:50,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:50,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:50,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:50,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:50,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:50,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:50,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +22: [2023-05-13 09:38:50,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:50,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:50,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +22: [2023-05-13 09:38:50,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:50,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:50,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +18: [2023-05-13 09:38:50,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:50,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:50,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +14: [2023-05-13 09:38:50,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:50,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:50,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:50,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:50,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:50,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:50,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:50,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +24: [2023-05-13 09:38:50,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +14: [2023-05-13 09:38:50,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:50,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:50,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:50,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:50,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:50,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:50,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:50,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:50,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +24: [2023-05-13 09:38:50,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 9: [2023-05-13 09:38:50,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:50,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 9: [2023-05-13 09:38:50,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +19: [2023-05-13 09:38:50,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +19: [2023-05-13 09:38:50,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +28: [2023-05-13 09:38:50,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:50,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:50,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 0: [2023-05-13 09:38:50,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:50,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:50,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:50,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +22: [2023-05-13 09:38:50,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +22: [2023-05-13 09:38:50,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:50,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:50,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:50,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:50,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:50,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:50,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:50,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +24: [2023-05-13 09:38:50,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:50,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:50,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:50,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:50,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:50,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:50,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:50,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:50,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:50,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:50,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +29: [2023-05-13 09:38:50,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:50,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:50,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +10: [2023-05-13 09:38:50,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +20: [2023-05-13 09:38:50,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:50,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:50,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:50,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:50,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:50,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:50,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:50,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +12: [2023-05-13 09:38:50,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:50,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:50,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:50,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:50,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:50,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:50,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:50,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 2: [2023-05-13 09:38:50,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 2: [2023-05-13 09:38:50,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 2: [2023-05-13 09:38:50,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +29: [2023-05-13 09:38:50,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:50,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +29: [2023-05-13 09:38:50,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:50,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:50,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:50,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 5: [2023-05-13 09:38:50,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:50,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:50,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +18: [2023-05-13 09:38:50,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:50,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:50,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:50,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:50,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:50,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:50,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 3: [2023-05-13 09:38:50,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +19: [2023-05-13 09:38:50,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:50,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:50,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:50,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +11: [2023-05-13 09:38:50,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 9: [2023-05-13 09:38:50,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:50,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:50,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:50,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:50,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:50,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:50,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:50,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 2: [2023-05-13 09:38:50,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:50,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +10: [2023-05-13 09:38:50,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:50,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:50,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:50,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:50,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:50,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:50,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:50,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:50,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:50,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:50,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +19: [2023-05-13 09:38:50,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +10: [2023-05-13 09:38:50,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:50,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:50,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:50,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:50,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:50,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:50,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:50,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:50,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:50,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:50,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +20: [2023-05-13 09:38:50,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +20: [2023-05-13 09:38:50,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:50,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:50,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:50,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:50,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:50,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:50,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:50,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:50,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:50,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:50,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:50,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:50,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:50,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:50,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:50,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +29: [2023-05-13 09:38:50,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:50,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:50,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:50,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +29: [2023-05-13 09:38:50,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +25: [2023-05-13 09:38:50,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:50,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 1: [2023-05-13 09:38:50,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:50,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:50,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:50,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:50,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +23: [2023-05-13 09:38:50,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +23: [2023-05-13 09:38:50,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:50,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:50,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:50,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +23: [2023-05-13 09:38:50,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:50,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 1: [2023-05-13 09:38:50,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:50,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:50,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:50,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:50,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:50,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:50,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:50,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:50,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +10: [2023-05-13 09:38:50,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:50,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:50,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +16: [2023-05-13 09:38:50,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:50,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:50,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +19: [2023-05-13 09:38:50,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:50,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:50,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +17: [2023-05-13 09:38:50,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:50,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:50,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 2: [2023-05-13 09:38:50,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:50,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:50,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 2: [2023-05-13 09:38:50,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 3: [2023-05-13 09:38:50,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:50,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:50,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +16: [2023-05-13 09:38:50,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:50,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:50,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:50,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:50,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:50,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +15: [2023-05-13 09:38:50,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:50,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:50,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +19: [2023-05-13 09:38:50,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:50,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:50,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:50,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:50,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:50,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:50,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:50,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 9: [2023-05-13 09:38:50,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:50,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:50,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:50,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:50,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:50,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:50,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:50,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:50,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +26: [2023-05-13 09:38:50,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:50,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:50,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:50,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:50,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:50,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:50,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:50,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:50,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:50,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +18: [2023-05-13 09:38:50,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:50,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:50,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:50,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:50,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:50,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:50,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:50,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:50,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:50,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +18: [2023-05-13 09:38:50,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:50,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:50,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:50,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:50,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:50,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:50,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:50,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:50,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:50,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:50,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:50,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:50,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:50,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:50,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:50,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:50,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:50,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:50,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:50,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:50,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:50,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:50,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:50,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:50,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:50,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:50,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:50,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:50,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:50,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:50,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:50,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:50,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:50,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +16: [2023-05-13 09:38:50,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +23: [2023-05-13 09:38:50,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +23: [2023-05-13 09:38:50,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:50,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:50,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:50,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:50,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:50,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +16: [2023-05-13 09:38:50,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:50,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 7: [2023-05-13 09:38:50,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 7: [2023-05-13 09:38:50,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 7: [2023-05-13 09:38:50,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:50,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:50,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:50,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +11: [2023-05-13 09:38:50,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:50,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +23: [2023-05-13 09:38:50,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:50,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:50,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +18: [2023-05-13 09:38:50,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:50,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:50,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +11: [2023-05-13 09:38:50,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +18: [2023-05-13 09:38:50,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:50,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:50,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:50,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +23: [2023-05-13 09:38:50,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:50,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +23: [2023-05-13 09:38:50,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:50,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:50,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:50,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:50,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:50,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 4: [2023-05-13 09:38:50,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +22: [2023-05-13 09:38:50,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +15: [2023-05-13 09:38:50,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:50,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:50,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:50,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:50,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:50,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:50,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 4: [2023-05-13 09:38:50,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 4: [2023-05-13 09:38:50,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 4: [2023-05-13 09:38:50,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 7: [2023-05-13 09:38:50,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:50,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 0: [2023-05-13 09:38:50,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 0: [2023-05-13 09:38:50,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +22: [2023-05-13 09:38:50,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:50,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:50,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:50,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +18: [2023-05-13 09:38:50,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +22: [2023-05-13 09:38:50,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:50,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:50,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:50,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:50,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:50,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:50,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:50,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:50,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +17: [2023-05-13 09:38:50,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:50,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:50,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +11: [2023-05-13 09:38:50,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:50,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +31: [2023-05-13 09:38:50,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:50,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:50,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:50,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:50,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:50,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:50,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:50,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:50,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +27: [2023-05-13 09:38:50,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:50,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:50,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +21: [2023-05-13 09:38:50,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +21: [2023-05-13 09:38:50,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:50,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:50,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:50,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:50,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:50,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 5: [2023-05-13 09:38:50,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 5: [2023-05-13 09:38:50,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 5: [2023-05-13 09:38:50,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 4: [2023-05-13 09:38:50,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:50,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:50,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:50,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:50,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +25: [2023-05-13 09:38:50,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +17: [2023-05-13 09:38:50,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:50,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +17: [2023-05-13 09:38:50,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:50,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:50,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:50,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +14: [2023-05-13 09:38:50,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:50,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +14: [2023-05-13 09:38:50,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:50,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +17: [2023-05-13 09:38:50,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +17: [2023-05-13 09:38:50,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +17: [2023-05-13 09:38:50,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:50,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:50,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:50,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +26: [2023-05-13 09:38:50,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:50,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:50,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:50,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:50,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:50,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +28: [2023-05-13 09:38:50,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +28: [2023-05-13 09:38:50,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +28: [2023-05-13 09:38:50,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 4: [2023-05-13 09:38:50,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:50,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:50,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:50,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:50,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:50,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:50,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:50,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:50,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:50,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +13: [2023-05-13 09:38:50,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +13: [2023-05-13 09:38:50,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +13: [2023-05-13 09:38:50,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. + 5: [2023-05-13 09:38:50,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:50,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:50,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:50,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:50,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +26: [2023-05-13 09:38:50,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:50,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:50,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:50,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:50,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +21: [2023-05-13 09:38:50,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:50,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:50,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:50,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:50,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:50,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:50,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:50,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:50,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:50,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:50,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:50,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:50,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:50,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:50,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:50,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:50,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:50,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +30: [2023-05-13 09:38:50,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_12-model_01-model_states.pt. +29: [2023-05-13 09:38:50,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:50,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:50,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 7: [2023-05-13 09:38:50,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:50,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:50,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 7: [2023-05-13 09:38:50,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 7: [2023-05-13 09:38:50,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +13: [2023-05-13 09:38:50,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:50,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:50,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:50,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:50,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:50,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:50,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +13: [2023-05-13 09:38:50,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +10: [2023-05-13 09:38:50,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:50,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:50,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:50,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:50,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:50,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:50,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:50,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:50,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:50,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:50,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:50,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:50,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:50,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:50,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:50,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:50,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:50,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:50,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +28: [2023-05-13 09:38:50,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:50,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:50,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:50,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +30: [2023-05-13 09:38:50,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt... +25: [2023-05-13 09:38:50,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +27: [2023-05-13 09:38:50,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:50,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:50,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:50,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +21: [2023-05-13 09:38:50,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:50,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:50,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:50,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:50,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:50,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:50,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:50,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:50,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:50,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:50,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:50,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +20: [2023-05-13 09:38:50,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:50,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:50,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:50,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 5: [2023-05-13 09:38:50,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +25: [2023-05-13 09:38:50,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +24: [2023-05-13 09:38:50,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:50,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:50,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +21: [2023-05-13 09:38:50,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:50,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:50,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:50,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:50,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:50,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:50,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:50,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:50,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +26: [2023-05-13 09:38:50,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:50,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:50,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 5: [2023-05-13 09:38:50,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 5: [2023-05-13 09:38:50,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 5: [2023-05-13 09:38:50,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +29: [2023-05-13 09:38:50,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:50,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:50,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:50,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:50,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +21: [2023-05-13 09:38:50,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:50,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:50,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +29: [2023-05-13 09:38:50,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:50,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:50,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:50,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:50,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:50,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:50,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:50,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:50,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:50,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:50,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:50,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:50,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:50,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +12: [2023-05-13 09:38:50,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:50,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:50,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:50,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:50,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:50,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 7: [2023-05-13 09:38:50,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:50,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:50,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +13: [2023-05-13 09:38:50,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:50,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:50,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:50,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:50,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:50,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +13: [2023-05-13 09:38:50,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +13: [2023-05-13 09:38:50,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +29: [2023-05-13 09:38:50,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:50,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:50,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:50,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:50,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... + 7: [2023-05-13 09:38:50,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:50,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +21: [2023-05-13 09:38:50,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:50,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:50,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:50,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:50,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:50,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:50,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +21: [2023-05-13 09:38:50,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:50,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:50,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:50,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:50,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:50,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:50,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:50,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:50,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:50,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +12: [2023-05-13 09:38:50,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:50,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:50,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:50,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:50,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:50,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:50,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:50,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:50,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:50,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +30: [2023-05-13 09:38:50,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:50,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +30: [2023-05-13 09:38:50,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_00-model_states.pt. +10: [2023-05-13 09:38:50,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:50,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:50,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:50,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:50,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +30: [2023-05-13 09:38:50,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt... +20: [2023-05-13 09:38:50,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:50,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:50,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:50,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:50,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:50,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:50,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:50,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:50,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:50,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +19: [2023-05-13 09:38:50,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +19: [2023-05-13 09:38:50,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +31: [2023-05-13 09:38:50,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +31: [2023-05-13 09:38:50,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +31: [2023-05-13 09:38:50,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +31: [2023-05-13 09:38:50,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +19: [2023-05-13 09:38:50,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +19: [2023-05-13 09:38:50,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +10: [2023-05-13 09:38:50,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +10: [2023-05-13 09:38:50,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 1: [2023-05-13 09:38:50,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 1: [2023-05-13 09:38:50,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 1: [2023-05-13 09:38:50,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 1: [2023-05-13 09:38:50,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +10: [2023-05-13 09:38:50,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +10: [2023-05-13 09:38:50,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:50,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:50,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:50,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:50,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:50,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +24: [2023-05-13 09:38:50,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +24: [2023-05-13 09:38:50,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +24: [2023-05-13 09:38:50,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 2: [2023-05-13 09:38:50,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 6: [2023-05-13 09:38:50,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 6: [2023-05-13 09:38:50,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +14: [2023-05-13 09:38:50,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:50,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:50,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:50,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:50,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:50,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:50,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:50,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:50,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:50,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:50,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:50,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:50,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:50,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:50,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:50,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +24: [2023-05-13 09:38:50,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:50,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +15: [2023-05-13 09:38:50,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:50,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:50,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:50,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:50,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:50,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:50,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:50,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:50,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:50,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:50,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:50,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:50,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +31: [2023-05-13 09:38:50,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:50,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +22: [2023-05-13 09:38:50,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:50,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:50,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +22: [2023-05-13 09:38:50,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:50,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:50,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:50,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:50,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:50,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:50,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +14: [2023-05-13 09:38:50,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:50,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:50,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:50,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:50,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:50,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:50,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:50,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:50,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:50,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +14: [2023-05-13 09:38:50,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:50,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +14: [2023-05-13 09:38:50,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:50,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:50,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +19: [2023-05-13 09:38:50,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +31: [2023-05-13 09:38:50,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:50,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:50,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:50,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:50,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:50,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:50,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +31: [2023-05-13 09:38:50,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:50,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +10: [2023-05-13 09:38:50,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:50,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +10: [2023-05-13 09:38:50,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +31: [2023-05-13 09:38:50,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +11: [2023-05-13 09:38:50,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:50,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:50,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:50,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:50,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +11: [2023-05-13 09:38:50,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:50,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:50,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:50,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:50,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:50,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:50,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:50,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:50,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +10: [2023-05-13 09:38:50,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +10: [2023-05-13 09:38:50,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:50,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:50,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:50,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:50,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:50,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:50,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:50,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:50,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:50,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:50,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:50,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:50,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:50,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:50,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:50,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:50,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:50,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:50,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:50,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:50,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:50,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:50,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +22: [2023-05-13 09:38:50,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:50,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:50,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:50,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 8: [2023-05-13 09:38:50,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:50,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +24: [2023-05-13 09:38:50,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:50,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:50,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:50,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:50,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:50,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:50,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +15: [2023-05-13 09:38:50,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:50,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:51,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +24: [2023-05-13 09:38:51,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +22: [2023-05-13 09:38:51,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:51,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +20: [2023-05-13 09:38:51,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +20: [2023-05-13 09:38:51,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +20: [2023-05-13 09:38:51,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +22: [2023-05-13 09:38:51,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:51,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:51,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:51,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 3: [2023-05-13 09:38:51,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 3: [2023-05-13 09:38:51,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:51,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:51,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:51,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:51,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:51,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:51,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:51,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:51,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:51,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:51,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 2: [2023-05-13 09:38:51,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:51,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 2: [2023-05-13 09:38:51,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 8: [2023-05-13 09:38:51,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:51,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:51,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:51,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:51,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +27: [2023-05-13 09:38:51,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:51,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:51,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +13: [2023-05-13 09:38:51,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 8: [2023-05-13 09:38:51,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 4: [2023-05-13 09:38:51,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +23: [2023-05-13 09:38:51,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +23: [2023-05-13 09:38:51,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +23: [2023-05-13 09:38:51,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +22: [2023-05-13 09:38:51,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 4: [2023-05-13 09:38:51,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 4: [2023-05-13 09:38:51,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +23: [2023-05-13 09:38:51,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +22: [2023-05-13 09:38:51,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 6: [2023-05-13 09:38:51,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:51,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +20: [2023-05-13 09:38:51,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:51,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:51,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:51,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:51,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:51,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:51,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:51,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:51,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:51,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 6: [2023-05-13 09:38:51,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:51,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 6: [2023-05-13 09:38:51,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:51,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:51,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:51,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:51,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:51,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +16: [2023-05-13 09:38:51,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +29: [2023-05-13 09:38:51,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:51,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:51,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:51,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:51,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:51,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +12: [2023-05-13 09:38:51,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +12: [2023-05-13 09:38:51,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +12: [2023-05-13 09:38:51,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +18: [2023-05-13 09:38:51,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:51,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:51,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +18: [2023-05-13 09:38:51,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:51,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:51,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:51,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:51,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:51,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:51,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +16: [2023-05-13 09:38:51,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:51,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:51,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:51,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:51,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:51,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:51,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:51,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:51,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:51,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:51,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:51,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:51,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:51,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:51,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:51,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:51,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:51,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:51,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:51,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:51,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:51,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:51,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:51,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:51,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:51,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +20: [2023-05-13 09:38:51,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:51,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:51,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +12: [2023-05-13 09:38:51,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:51,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:51,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +20: [2023-05-13 09:38:51,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:51,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:51,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:51,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:51,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:51,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +20: [2023-05-13 09:38:51,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +20: [2023-05-13 09:38:51,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:51,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:51,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:51,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 6: [2023-05-13 09:38:51,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:51,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:51,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:51,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +29: [2023-05-13 09:38:51,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +18: [2023-05-13 09:38:51,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:51,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +22: [2023-05-13 09:38:51,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +27: [2023-05-13 09:38:51,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:51,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:51,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +27: [2023-05-13 09:38:51,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +29: [2023-05-13 09:38:51,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +27: [2023-05-13 09:38:51,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +16: [2023-05-13 09:38:51,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:51,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:51,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +27: [2023-05-13 09:38:51,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +29: [2023-05-13 09:38:51,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +23: [2023-05-13 09:38:51,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:51,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +22: [2023-05-13 09:38:51,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +16: [2023-05-13 09:38:51,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +16: [2023-05-13 09:38:51,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +27: [2023-05-13 09:38:51,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +16: [2023-05-13 09:38:51,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +27: [2023-05-13 09:38:51,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:51,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:51,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:51,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:51,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +23: [2023-05-13 09:38:51,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:51,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +22: [2023-05-13 09:38:51,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +11: [2023-05-13 09:38:51,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +22: [2023-05-13 09:38:51,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:51,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +18: [2023-05-13 09:38:51,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +18: [2023-05-13 09:38:51,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +27: [2023-05-13 09:38:51,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +23: [2023-05-13 09:38:51,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:51,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:51,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:51,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:51,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:51,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:51,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:51,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 9: [2023-05-13 09:38:51,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +23: [2023-05-13 09:38:51,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:51,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:51,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:51,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:51,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +23: [2023-05-13 09:38:51,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +28: [2023-05-13 09:38:51,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:51,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:51,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:51,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:51,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +12: [2023-05-13 09:38:51,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +22: [2023-05-13 09:38:51,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +12: [2023-05-13 09:38:51,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:51,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:51,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 5: [2023-05-13 09:38:51,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 5: [2023-05-13 09:38:51,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +11: [2023-05-13 09:38:51,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 5: [2023-05-13 09:38:51,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:51,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +25: [2023-05-13 09:38:51,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 5: [2023-05-13 09:38:51,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:51,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:51,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 5: [2023-05-13 09:38:51,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:51,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:51,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:51,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +11: [2023-05-13 09:38:51,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:51,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:51,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +11: [2023-05-13 09:38:51,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 0: [2023-05-13 09:38:51,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 0: [2023-05-13 09:38:51,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 0: [2023-05-13 09:38:51,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:51,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:51,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:51,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:51,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:51,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:51,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:51,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:51,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:51,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:51,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +25: [2023-05-13 09:38:51,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:51,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:51,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +11: [2023-05-13 09:38:51,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +13: [2023-05-13 09:38:51,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +13: [2023-05-13 09:38:51,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +13: [2023-05-13 09:38:51,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +13: [2023-05-13 09:38:51,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 5: [2023-05-13 09:38:51,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:51,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:51,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:51,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:51,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:51,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:51,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +28: [2023-05-13 09:38:51,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:51,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:51,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:51,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:51,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:51,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +13: [2023-05-13 09:38:51,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:51,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:51,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:51,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:51,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +28: [2023-05-13 09:38:51,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:51,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +17: [2023-05-13 09:38:51,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:51,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:51,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:51,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:51,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:51,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:51,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +15: [2023-05-13 09:38:51,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:51,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:51,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:51,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:51,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:51,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:51,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:51,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:51,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:51,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:51,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:51,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:51,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:51,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:51,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:51,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +23: [2023-05-13 09:38:51,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:51,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:51,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:51,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:51,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:51,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +25: [2023-05-13 09:38:51,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +25: [2023-05-13 09:38:51,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +26: [2023-05-13 09:38:51,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:51,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:51,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:51,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 5: [2023-05-13 09:38:51,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +28: [2023-05-13 09:38:51,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:51,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +13: [2023-05-13 09:38:51,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:51,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +18: [2023-05-13 09:38:51,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:51,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +13: [2023-05-13 09:38:51,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:51,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +19: [2023-05-13 09:38:51,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:51,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +13: [2023-05-13 09:38:51,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +18: [2023-05-13 09:38:51,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:51,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +28: [2023-05-13 09:38:51,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +28: [2023-05-13 09:38:51,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +13: [2023-05-13 09:38:51,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +17: [2023-05-13 09:38:51,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:51,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +15: [2023-05-13 09:38:51,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:51,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:51,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... + 1: [2023-05-13 09:38:51,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:51,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:51,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:51,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:51,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:51,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:51,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +30: [2023-05-13 09:38:51,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +30: [2023-05-13 09:38:51,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +30: [2023-05-13 09:38:51,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_13-model_01-model_states.pt. +17: [2023-05-13 09:38:51,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 7: [2023-05-13 09:38:51,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 7: [2023-05-13 09:38:51,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 7: [2023-05-13 09:38:51,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +15: [2023-05-13 09:38:51,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:51,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +26: [2023-05-13 09:38:51,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +15: [2023-05-13 09:38:51,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:51,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +26: [2023-05-13 09:38:51,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +19: [2023-05-13 09:38:51,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +19: [2023-05-13 09:38:51,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:51,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:51,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +19: [2023-05-13 09:38:51,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +19: [2023-05-13 09:38:51,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:51,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:51,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:51,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:51,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:51,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:51,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:51,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +30: [2023-05-13 09:38:51,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt... +26: [2023-05-13 09:38:51,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:51,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:51,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:51,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:51,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:51,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:51,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:51,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:51,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:51,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 1: [2023-05-13 09:38:51,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:51,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:51,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:51,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +26: [2023-05-13 09:38:51,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:51,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:51,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:51,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:51,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:51,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:51,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +19: [2023-05-13 09:38:51,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +26: [2023-05-13 09:38:51,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:51,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:51,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:51,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:51,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +26: [2023-05-13 09:38:51,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:51,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:51,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +19: [2023-05-13 09:38:51,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:51,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +30: [2023-05-13 09:38:51,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_00-model_states.pt. +20: [2023-05-13 09:38:51,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:51,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:51,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:51,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:51,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:51,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:51,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:51,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:51,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:51,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt... +20: [2023-05-13 09:38:51,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:51,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:51,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:51,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:51,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 1: [2023-05-13 09:38:51,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 1: [2023-05-13 09:38:51,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +19: [2023-05-13 09:38:51,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +26: [2023-05-13 09:38:51,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:51,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:51,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:51,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:51,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:51,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:51,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:51,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:51,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +31: [2023-05-13 09:38:51,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:51,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:51,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:51,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:51,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:51,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:51,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +15: [2023-05-13 09:38:51,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:51,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:51,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:51,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:51,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +15: [2023-05-13 09:38:51,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:51,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:51,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:51,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:51,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:51,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:51,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:51,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:51,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:51,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:51,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:51,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:51,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:51,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:51,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:51,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:51,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:51,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +12: [2023-05-13 09:38:51,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:51,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:51,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:51,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:51,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:51,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:51,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:51,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:51,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:51,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:51,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:51,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:51,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:51,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +29: [2023-05-13 09:38:51,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +11: [2023-05-13 09:38:51,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:51,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:51,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:51,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:51,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +11: [2023-05-13 09:38:51,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:51,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:51,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:51,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 9: [2023-05-13 09:38:51,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:51,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:51,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:51,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:51,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:51,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:51,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:51,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:51,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:51,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:51,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:51,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 6: [2023-05-13 09:38:51,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 6: [2023-05-13 09:38:51,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +15: [2023-05-13 09:38:51,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:51,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:51,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +29: [2023-05-13 09:38:51,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:51,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:51,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +15: [2023-05-13 09:38:51,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +29: [2023-05-13 09:38:51,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:51,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:51,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:51,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:51,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +18: [2023-05-13 09:38:51,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +30: [2023-05-13 09:38:51,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:51,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:51,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:51,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:51,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:51,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:51,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:51,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:51,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:51,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:51,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:51,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:51,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:51,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:51,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:51,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:51,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +12: [2023-05-13 09:38:51,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:51,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:51,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:51,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:51,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +11: [2023-05-13 09:38:51,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:51,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:51,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:51,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:51,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:51,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:51,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:51,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:51,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:51,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +22: [2023-05-13 09:38:51,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +22: [2023-05-13 09:38:51,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +24: [2023-05-13 09:38:51,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +24: [2023-05-13 09:38:51,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:51,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:51,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:51,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:51,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:51,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:51,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +14: [2023-05-13 09:38:51,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +14: [2023-05-13 09:38:51,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +14: [2023-05-13 09:38:51,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +13: [2023-05-13 09:38:51,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:51,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +29: [2023-05-13 09:38:51,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:51,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 6: [2023-05-13 09:38:51,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +29: [2023-05-13 09:38:51,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +29: [2023-05-13 09:38:51,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:51,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:51,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:51,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:51,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:51,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:51,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:51,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:51,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:51,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:51,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:51,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:51,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 6: [2023-05-13 09:38:51,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:51,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:51,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:51,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +18: [2023-05-13 09:38:51,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:51,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:51,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:51,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:51,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:51,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +28: [2023-05-13 09:38:51,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:51,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +28: [2023-05-13 09:38:51,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:51,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 7: [2023-05-13 09:38:51,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +10: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:51,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +18: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:51,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:51,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +31: [2023-05-13 09:38:51,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +25: [2023-05-13 09:38:51,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:51,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:51,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:51,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:51,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +20: [2023-05-13 09:38:51,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +18: [2023-05-13 09:38:51,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +18: [2023-05-13 09:38:51,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +20: [2023-05-13 09:38:51,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 0: [2023-05-13 09:38:51,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:51,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 0: [2023-05-13 09:38:51,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:51,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +25: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 4: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +25: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:51,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 5: [2023-05-13 09:38:51,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 5: [2023-05-13 09:38:51,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:51,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +22: [2023-05-13 09:38:51,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +24: [2023-05-13 09:38:51,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +14: [2023-05-13 09:38:51,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:51,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:51,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:51,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 5: [2023-05-13 09:38:51,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +22: [2023-05-13 09:38:51,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:51,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +14: [2023-05-13 09:38:51,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:51,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:51,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:51,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:51,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:51,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:51,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:51,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:51,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:51,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:51,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:51,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +14: [2023-05-13 09:38:51,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:51,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +11: [2023-05-13 09:38:51,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 3: [2023-05-13 09:38:51,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:51,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:51,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +10: [2023-05-13 09:38:51,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:51,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +31: [2023-05-13 09:38:51,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:51,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:51,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:51,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +14: [2023-05-13 09:38:51,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:51,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:51,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +28: [2023-05-13 09:38:51,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:51,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:51,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +10: [2023-05-13 09:38:51,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 5: [2023-05-13 09:38:51,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 5: [2023-05-13 09:38:51,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:51,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +19: [2023-05-13 09:38:51,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 5: [2023-05-13 09:38:51,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 0: [2023-05-13 09:38:51,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:51,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:51,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:51,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +19: [2023-05-13 09:38:51,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:51,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:51,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:51,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:51,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +11: [2023-05-13 09:38:51,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +11: [2023-05-13 09:38:51,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +18: [2023-05-13 09:38:51,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:51,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +16: [2023-05-13 09:38:51,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:51,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:51,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:51,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:51,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:51,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:51,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:51,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +16: [2023-05-13 09:38:51,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +20: [2023-05-13 09:38:51,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 2: [2023-05-13 09:38:51,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 2: [2023-05-13 09:38:51,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 2: [2023-05-13 09:38:51,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +26: [2023-05-13 09:38:51,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:51,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +25: [2023-05-13 09:38:51,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +25: [2023-05-13 09:38:51,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +25: [2023-05-13 09:38:51,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +16: [2023-05-13 09:38:51,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:51,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:51,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:51,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +23: [2023-05-13 09:38:51,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:51,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:51,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:51,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +16: [2023-05-13 09:38:51,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:51,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +26: [2023-05-13 09:38:51,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:51,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:51,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:51,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:51,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +20: [2023-05-13 09:38:51,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:51,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:51,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +18: [2023-05-13 09:38:51,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:51,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:51,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:51,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:51,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:51,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:51,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:51,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:51,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:51,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +16: [2023-05-13 09:38:51,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:51,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:51,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +23: [2023-05-13 09:38:51,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 6: [2023-05-13 09:38:51,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:51,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +23: [2023-05-13 09:38:51,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 3: [2023-05-13 09:38:51,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +23: [2023-05-13 09:38:51,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +16: [2023-05-13 09:38:51,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +16: [2023-05-13 09:38:51,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +16: [2023-05-13 09:38:51,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +18: [2023-05-13 09:38:51,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:51,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:51,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:51,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 3: [2023-05-13 09:38:51,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 2: [2023-05-13 09:38:51,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 6: [2023-05-13 09:38:51,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:51,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +20: [2023-05-13 09:38:51,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:51,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:51,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:51,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:51,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:51,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:51,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:51,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +12: [2023-05-13 09:38:51,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +17: [2023-05-13 09:38:51,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:51,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:51,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:51,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:51,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:51,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 2: [2023-05-13 09:38:51,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 9: [2023-05-13 09:38:51,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:51,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +28: [2023-05-13 09:38:51,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +17: [2023-05-13 09:38:51,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:51,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +25: [2023-05-13 09:38:51,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:51,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 8: [2023-05-13 09:38:51,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 3: [2023-05-13 09:38:51,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +25: [2023-05-13 09:38:51,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +25: [2023-05-13 09:38:51,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +25: [2023-05-13 09:38:51,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:51,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:51,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +25: [2023-05-13 09:38:51,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +25: [2023-05-13 09:38:51,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 9: [2023-05-13 09:38:51,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:51,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 9: [2023-05-13 09:38:51,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:51,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:51,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:51,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:51,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:51,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:51,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:51,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:51,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:51,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:51,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +13: [2023-05-13 09:38:51,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +13: [2023-05-13 09:38:51,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:51,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:51,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:51,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +27: [2023-05-13 09:38:51,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:51,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:51,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:51,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:51,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +28: [2023-05-13 09:38:51,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +30: [2023-05-13 09:38:51,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +30: [2023-05-13 09:38:51,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +28: [2023-05-13 09:38:51,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +28: [2023-05-13 09:38:51,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_14-model_01-model_states.pt. +21: [2023-05-13 09:38:51,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:51,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:51,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +21: [2023-05-13 09:38:51,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 3: [2023-05-13 09:38:51,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +17: [2023-05-13 09:38:51,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:51,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:51,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:51,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:51,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:51,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:51,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:51,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:51,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:51,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:51,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:51,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:51,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +15: [2023-05-13 09:38:51,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:51,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:51,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:51,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:51,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:51,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:51,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +29: [2023-05-13 09:38:51,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +12: [2023-05-13 09:38:51,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:51,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +13: [2023-05-13 09:38:51,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +29: [2023-05-13 09:38:51,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +13: [2023-05-13 09:38:51,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +12: [2023-05-13 09:38:51,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +13: [2023-05-13 09:38:51,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +30: [2023-05-13 09:38:51,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:51,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +21: [2023-05-13 09:38:51,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:51,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:51,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:51,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 0: [2023-05-13 09:38:51,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:51,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:51,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... +27: [2023-05-13 09:38:51,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 5: [2023-05-13 09:38:51,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:51,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:51,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:51,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:51,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +31: [2023-05-13 09:38:51,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:51,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +17: [2023-05-13 09:38:51,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:51,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:51,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:51,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +26: [2023-05-13 09:38:51,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +26: [2023-05-13 09:38:51,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +26: [2023-05-13 09:38:51,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:51,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 8: [2023-05-13 09:38:51,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 1: [2023-05-13 09:38:51,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:51,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +27: [2023-05-13 09:38:51,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +27: [2023-05-13 09:38:51,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:51,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:51,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:51,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +31: [2023-05-13 09:38:51,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:51,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:51,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:51,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +20: [2023-05-13 09:38:51,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +20: [2023-05-13 09:38:51,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +20: [2023-05-13 09:38:51,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 1: [2023-05-13 09:38:51,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +31: [2023-05-13 09:38:51,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:51,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:51,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:51,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:51,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +29: [2023-05-13 09:38:51,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +13: [2023-05-13 09:38:51,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:51,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:51,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:51,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:51,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:51,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +27: [2023-05-13 09:38:51,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:51,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:51,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +15: [2023-05-13 09:38:51,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:51,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:51,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:51,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:51,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:51,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:51,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +24: [2023-05-13 09:38:51,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +24: [2023-05-13 09:38:51,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 5: [2023-05-13 09:38:51,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:51,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:51,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:51,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:51,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:51,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:51,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:51,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:51,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 1: [2023-05-13 09:38:51,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:51,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:51,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:51,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:51,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:51,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 1: [2023-05-13 09:38:51,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:51,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +16: [2023-05-13 09:38:51,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:51,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:51,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:51,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +22: [2023-05-13 09:38:51,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:51,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:51,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:51,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:51,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:51,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:51,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... + 1: [2023-05-13 09:38:51,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:51,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:51,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:51,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:51,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:51,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +21: [2023-05-13 09:38:51,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:51,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +21: [2023-05-13 09:38:51,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +30: [2023-05-13 09:38:51,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +21: [2023-05-13 09:38:51,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:51,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:51,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:51,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +29: [2023-05-13 09:38:51,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:51,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:51,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:51,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:51,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:51,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +30: [2023-05-13 09:38:51,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +30: [2023-05-13 09:38:51,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:51,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +13: [2023-05-13 09:38:51,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:51,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_00-model_states.pt. +24: [2023-05-13 09:38:51,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:51,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:51,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +30: [2023-05-13 09:38:51,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt... +10: [2023-05-13 09:38:51,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:51,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:51,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:51,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:51,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:51,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:51,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:51,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:51,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:51,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +26: [2023-05-13 09:38:51,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +26: [2023-05-13 09:38:51,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +26: [2023-05-13 09:38:51,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +11: [2023-05-13 09:38:51,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:51,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:51,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:51,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +30: [2023-05-13 09:38:51,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +20: [2023-05-13 09:38:51,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +20: [2023-05-13 09:38:51,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +20: [2023-05-13 09:38:51,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +22: [2023-05-13 09:38:51,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:51,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:51,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:51,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:51,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:51,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:51,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:51,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:51,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:51,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:51,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:51,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:51,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:51,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:51,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:51,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:51,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:51,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:51,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:51,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:51,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:51,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:51,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:51,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:51,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:51,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:51,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:51,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:51,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +19: [2023-05-13 09:38:51,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +10: [2023-05-13 09:38:51,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:51,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:51,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:51,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:51,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:51,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:51,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +22: [2023-05-13 09:38:51,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +22: [2023-05-13 09:38:51,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +22: [2023-05-13 09:38:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +15: [2023-05-13 09:38:51,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +15: [2023-05-13 09:38:51,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 6: [2023-05-13 09:38:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 6: [2023-05-13 09:38:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 5: [2023-05-13 09:38:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 5: [2023-05-13 09:38:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 5: [2023-05-13 09:38:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 6: [2023-05-13 09:38:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 6: [2023-05-13 09:38:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 5: [2023-05-13 09:38:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +14: [2023-05-13 09:38:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +14: [2023-05-13 09:38:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +14: [2023-05-13 09:38:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +18: [2023-05-13 09:38:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +18: [2023-05-13 09:38:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +14: [2023-05-13 09:38:51,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +18: [2023-05-13 09:38:51,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +18: [2023-05-13 09:38:51,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:51,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:51,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:51,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +29: [2023-05-13 09:38:51,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +10: [2023-05-13 09:38:51,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:51,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:51,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:51,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:51,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:51,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:51,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:51,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:51,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:51,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:51,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 6: [2023-05-13 09:38:51,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +29: [2023-05-13 09:38:51,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +10: [2023-05-13 09:38:51,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +10: [2023-05-13 09:38:51,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +10: [2023-05-13 09:38:51,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +10: [2023-05-13 09:38:51,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:51,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:51,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:51,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:51,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:51,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:51,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:51,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:51,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:51,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:51,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:51,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:51,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:51,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:51,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:51,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:51,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +11: [2023-05-13 09:38:51,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +15: [2023-05-13 09:38:51,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:51,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +22: [2023-05-13 09:38:51,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:51,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:51,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:51,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:51,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:51,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +23: [2023-05-13 09:38:51,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +11: [2023-05-13 09:38:51,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:51,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +22: [2023-05-13 09:38:51,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:51,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:51,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +22: [2023-05-13 09:38:51,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:51,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:51,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:51,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +15: [2023-05-13 09:38:51,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:51,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:51,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 6: [2023-05-13 09:38:51,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:51,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:51,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +15: [2023-05-13 09:38:51,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +15: [2023-05-13 09:38:51,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 6: [2023-05-13 09:38:51,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 5: [2023-05-13 09:38:51,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 6: [2023-05-13 09:38:51,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 5: [2023-05-13 09:38:51,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 2: [2023-05-13 09:38:51,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +14: [2023-05-13 09:38:51,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:51,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 4: [2023-05-13 09:38:51,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +29: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:51,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +14: [2023-05-13 09:38:51,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +25: [2023-05-13 09:38:51,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:51,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:51,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 7: [2023-05-13 09:38:51,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 7: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:51,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 5: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 5: [2023-05-13 09:38:51,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 7: [2023-05-13 09:38:51,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +29: [2023-05-13 09:38:51,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 4: [2023-05-13 09:38:51,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:51,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +18: [2023-05-13 09:38:51,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:51,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:51,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:51,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:51,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 7: [2023-05-13 09:38:51,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 5: [2023-05-13 09:38:51,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 5: [2023-05-13 09:38:51,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 5: [2023-05-13 09:38:51,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +29: [2023-05-13 09:38:51,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:51,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 7: [2023-05-13 09:38:51,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 7: [2023-05-13 09:38:51,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:51,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:51,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:51,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:51,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:51,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:51,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:51,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:51,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:51,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:51,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:51,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:51,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:51,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:51,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:51,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:51,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:51,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:51,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:51,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:51,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:51,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:51,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 4: [2023-05-13 09:38:51,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:51,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:51,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:51,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:51,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +31: [2023-05-13 09:38:51,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +31: [2023-05-13 09:38:51,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:51,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 4: [2023-05-13 09:38:51,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:51,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:51,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:51,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:51,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:51,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:51,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:51,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:51,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +19: [2023-05-13 09:38:51,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:51,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:51,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:51,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:51,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:51,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:51,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:51,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:51,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:51,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:51,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:51,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +11: [2023-05-13 09:38:51,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 8: [2023-05-13 09:38:51,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 8: [2023-05-13 09:38:51,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 8: [2023-05-13 09:38:51,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 8: [2023-05-13 09:38:51,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:51,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 9: [2023-05-13 09:38:51,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:51,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 3: [2023-05-13 09:38:51,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 3: [2023-05-13 09:38:51,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 3: [2023-05-13 09:38:51,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +10: [2023-05-13 09:38:51,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +11: [2023-05-13 09:38:51,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:51,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +11: [2023-05-13 09:38:51,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +10: [2023-05-13 09:38:51,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:51,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:51,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:51,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:51,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +10: [2023-05-13 09:38:51,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:51,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:51,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:51,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 7: [2023-05-13 09:38:51,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:51,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +24: [2023-05-13 09:38:51,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +14: [2023-05-13 09:38:51,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:51,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:51,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:51,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:51,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:51,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +12: [2023-05-13 09:38:51,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:51,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +24: [2023-05-13 09:38:51,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:51,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:51,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +24: [2023-05-13 09:38:51,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:51,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +16: [2023-05-13 09:38:51,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 7: [2023-05-13 09:38:51,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:51,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +16: [2023-05-13 09:38:51,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +25: [2023-05-13 09:38:51,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:51,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 2: [2023-05-13 09:38:51,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:51,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:51,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:51,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:51,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:51,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:51,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:51,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:51,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 7: [2023-05-13 09:38:51,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:51,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:51,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:51,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:51,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:51,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:51,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:51,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:51,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:51,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:51,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:51,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:51,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:51,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:51,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:51,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +14: [2023-05-13 09:38:51,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:51,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:51,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +31: [2023-05-13 09:38:51,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:51,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +31: [2023-05-13 09:38:51,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:51,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +31: [2023-05-13 09:38:51,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:51,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +23: [2023-05-13 09:38:51,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:51,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:51,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +25: [2023-05-13 09:38:51,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +23: [2023-05-13 09:38:51,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +23: [2023-05-13 09:38:51,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +23: [2023-05-13 09:38:51,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:51,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:51,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 2: [2023-05-13 09:38:51,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:51,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:51,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:51,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:51,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +19: [2023-05-13 09:38:51,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:51,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +19: [2023-05-13 09:38:51,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:51,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 8: [2023-05-13 09:38:51,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:51,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 3: [2023-05-13 09:38:51,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 8: [2023-05-13 09:38:51,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:51,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:51,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 8: [2023-05-13 09:38:51,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +23: [2023-05-13 09:38:51,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:51,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:51,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:51,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:51,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +13: [2023-05-13 09:38:51,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +13: [2023-05-13 09:38:51,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +25: [2023-05-13 09:38:51,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:51,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 9: [2023-05-13 09:38:51,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +12: [2023-05-13 09:38:51,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +18: [2023-05-13 09:38:51,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:51,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +16: [2023-05-13 09:38:51,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:51,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 3: [2023-05-13 09:38:51,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +12: [2023-05-13 09:38:51,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +12: [2023-05-13 09:38:51,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 3: [2023-05-13 09:38:51,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:51,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:51,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:51,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +30: [2023-05-13 09:38:51,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +30: [2023-05-13 09:38:51,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +30: [2023-05-13 09:38:51,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +16: [2023-05-13 09:38:51,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +16: [2023-05-13 09:38:51,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:51,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +27: [2023-05-13 09:38:51,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +27: [2023-05-13 09:38:51,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +27: [2023-05-13 09:38:51,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +16: [2023-05-13 09:38:51,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +13: [2023-05-13 09:38:51,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +16: [2023-05-13 09:38:51,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +13: [2023-05-13 09:38:51,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +13: [2023-05-13 09:38:51,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. + 0: [2023-05-13 09:38:51,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +16: [2023-05-13 09:38:51,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +18: [2023-05-13 09:38:51,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:51,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:51,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:51,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:51,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +25: [2023-05-13 09:38:51,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:51,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +28: [2023-05-13 09:38:51,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +25: [2023-05-13 09:38:51,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:51,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:51,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:51,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:51,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:51,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:51,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:51,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:51,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:51,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +27: [2023-05-13 09:38:51,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:51,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +30: [2023-05-13 09:38:51,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +17: [2023-05-13 09:38:51,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:51,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +17: [2023-05-13 09:38:51,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:51,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_15-model_01-model_states.pt. +27: [2023-05-13 09:38:51,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:51,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +25: [2023-05-13 09:38:51,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +25: [2023-05-13 09:38:51,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:51,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +25: [2023-05-13 09:38:51,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +21: [2023-05-13 09:38:51,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:51,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:51,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:51,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +21: [2023-05-13 09:38:51,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:51,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:51,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:51,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:51,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:51,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +13: [2023-05-13 09:38:51,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +13: [2023-05-13 09:38:51,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:51,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... +28: [2023-05-13 09:38:51,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt... + 9: [2023-05-13 09:38:51,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 9: [2023-05-13 09:38:51,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 9: [2023-05-13 09:38:51,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +13: [2023-05-13 09:38:51,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 9: [2023-05-13 09:38:51,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +30: [2023-05-13 09:38:51,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +13: [2023-05-13 09:38:51,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:51,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:51,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:51,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +13: [2023-05-13 09:38:51,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +27: [2023-05-13 09:38:51,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +27: [2023-05-13 09:38:51,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:51,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:51,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... + 0: [2023-05-13 09:38:51,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +27: [2023-05-13 09:38:51,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +27: [2023-05-13 09:38:51,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:51,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:51,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:51,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +21: [2023-05-13 09:38:51,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:51,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:51,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:51,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +17: [2023-05-13 09:38:51,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +30: [2023-05-13 09:38:51,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +30: [2023-05-13 09:38:51,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +21: [2023-05-13 09:38:51,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:51,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:51,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +27: [2023-05-13 09:38:51,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +21: [2023-05-13 09:38:51,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:51,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +27: [2023-05-13 09:38:51,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +17: [2023-05-13 09:38:51,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:51,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +26: [2023-05-13 09:38:51,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:51,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:51,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:51,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:51,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:51,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:51,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +26: [2023-05-13 09:38:51,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:51,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:51,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:51,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:51,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:51,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. +28: [2023-05-13 09:38:51,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_00-model_states.pt. + 0: [2023-05-13 09:38:51,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:51,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +28: [2023-05-13 09:38:51,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt... +21: [2023-05-13 09:38:51,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:51,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:51,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:52,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:52,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:52,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:52,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:52,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:52,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:52,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:52,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:52,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:52,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:52,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:52,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:52,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:52,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:52,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:52,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:52,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:52,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +20: [2023-05-13 09:38:52,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:52,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:52,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:52,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:52,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +20: [2023-05-13 09:38:52,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +27: [2023-05-13 09:38:52,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:52,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:52,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:52,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:52,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:52,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:52,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:52,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:52,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:52,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:52,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +26: [2023-05-13 09:38:52,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:52,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:52,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:52,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:52,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:52,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:52,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +15: [2023-05-13 09:38:52,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:52,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:52,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:52,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:52,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:52,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:52,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:52,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:52,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:52,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:52,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +22: [2023-05-13 09:38:52,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +22: [2023-05-13 09:38:52,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +22: [2023-05-13 09:38:52,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 1: [2023-05-13 09:38:52,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:52,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:52,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:52,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:52,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:52,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:52,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:52,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:52,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +19: [2023-05-13 09:38:52,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +13: [2023-05-13 09:38:52,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:52,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +20: [2023-05-13 09:38:52,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:52,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +22: [2023-05-13 09:38:52,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:52,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:52,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:52,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:52,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +26: [2023-05-13 09:38:52,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:52,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +22: [2023-05-13 09:38:52,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:52,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:52,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:52,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:52,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:52,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 5: [2023-05-13 09:38:52,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +20: [2023-05-13 09:38:52,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:52,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +26: [2023-05-13 09:38:52,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:52,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:52,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:52,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:52,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:52,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:52,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:52,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:52,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:52,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:52,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:52,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:52,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:52,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:52,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:52,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:52,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:52,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:52,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:52,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:52,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:52,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:52,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:52,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:52,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:52,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:52,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:52,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:52,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:52,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +19: [2023-05-13 09:38:52,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:52,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:52,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:52,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:52,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:52,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:52,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:52,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:52,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:52,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:52,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:52,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:52,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:52,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:52,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +19: [2023-05-13 09:38:52,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +24: [2023-05-13 09:38:52,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:52,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:52,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:52,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:52,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 9: [2023-05-13 09:38:52,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:52,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:52,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:52,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +29: [2023-05-13 09:38:52,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:52,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +29: [2023-05-13 09:38:52,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +12: [2023-05-13 09:38:52,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +29: [2023-05-13 09:38:52,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:52,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +12: [2023-05-13 09:38:52,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:52,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:52,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:52,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:52,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:52,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:52,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:52,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:52,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 7: [2023-05-13 09:38:52,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:52,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:52,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:52,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:52,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:52,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +13: [2023-05-13 09:38:52,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 4: [2023-05-13 09:38:52,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +30: [2023-05-13 09:38:52,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:52,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:52,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:52,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:52,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:52,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:52,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:52,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:52,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:52,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:52,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:52,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:52,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:52,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:52,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:52,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:52,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:52,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:52,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:52,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:52,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:52,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:52,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:52,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +15: [2023-05-13 09:38:52,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +13: [2023-05-13 09:38:52,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 0: [2023-05-13 09:38:52,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:52,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:52,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:52,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:52,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:52,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:52,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:52,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:52,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:52,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:52,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:52,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:52,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:52,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:52,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:52,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:52,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:52,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:52,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:52,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +25: [2023-05-13 09:38:52,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:52,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:52,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:52,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:52,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:52,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:52,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:52,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:52,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:52,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:52,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:52,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:52,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 1: [2023-05-13 09:38:52,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 1: [2023-05-13 09:38:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +23: [2023-05-13 09:38:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +25: [2023-05-13 09:38:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +29: [2023-05-13 09:38:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:52,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:52,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:52,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +29: [2023-05-13 09:38:52,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +21: [2023-05-13 09:38:52,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:52,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:52,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:52,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:52,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +18: [2023-05-13 09:38:52,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:52,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:52,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:52,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:52,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:52,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:52,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:52,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:52,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:52,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:52,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +18: [2023-05-13 09:38:52,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +18: [2023-05-13 09:38:52,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:52,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:52,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:52,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:52,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:52,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:52,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:52,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +21: [2023-05-13 09:38:52,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:52,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:52,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:52,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:52,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:52,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +15: [2023-05-13 09:38:52,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 7: [2023-05-13 09:38:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 7: [2023-05-13 09:38:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 7: [2023-05-13 09:38:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:52,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:52,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +15: [2023-05-13 09:38:52,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:52,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:52,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:52,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:52,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:52,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:52,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:52,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:52,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:52,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:52,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:52,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +23: [2023-05-13 09:38:52,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:52,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +15: [2023-05-13 09:38:52,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +25: [2023-05-13 09:38:52,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:52,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 2: [2023-05-13 09:38:52,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 2: [2023-05-13 09:38:52,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:52,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:52,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:52,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:52,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:52,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:52,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:52,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:52,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:52,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:52,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:52,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 1: [2023-05-13 09:38:52,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +21: [2023-05-13 09:38:52,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:52,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:52,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:52,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:52,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:52,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:52,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +25: [2023-05-13 09:38:52,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:52,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:52,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:52,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:52,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:52,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +27: [2023-05-13 09:38:52,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +27: [2023-05-13 09:38:52,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:52,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +21: [2023-05-13 09:38:52,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:52,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:52,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +25: [2023-05-13 09:38:52,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:52,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:52,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +21: [2023-05-13 09:38:52,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:52,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:52,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:52,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:52,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:52,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:52,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:52,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:52,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:52,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:52,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:52,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +19: [2023-05-13 09:38:52,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:52,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:52,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:52,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:52,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:52,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +19: [2023-05-13 09:38:52,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:52,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:52,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:52,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:52,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +27: [2023-05-13 09:38:52,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:52,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:52,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:52,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:52,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +20: [2023-05-13 09:38:52,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:52,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:52,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:52,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +25: [2023-05-13 09:38:52,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +25: [2023-05-13 09:38:52,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:52,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:52,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:52,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:52,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:52,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:52,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:52,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:52,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:52,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:52,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:52,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:52,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:52,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:52,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +25: [2023-05-13 09:38:52,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +20: [2023-05-13 09:38:52,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:52,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:52,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:52,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +10: [2023-05-13 09:38:52,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +10: [2023-05-13 09:38:52,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +19: [2023-05-13 09:38:52,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:52,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +29: [2023-05-13 09:38:52,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:52,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:52,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:52,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:52,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:52,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:52,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +30: [2023-05-13 09:38:52,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +30: [2023-05-13 09:38:52,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +30: [2023-05-13 09:38:52,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +20: [2023-05-13 09:38:52,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:52,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:52,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:52,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:52,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:52,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:52,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:52,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:52,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:52,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:52,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:52,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:52,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:52,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:52,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:52,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:52,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:52,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:52,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:52,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:52,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:52,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +16: [2023-05-13 09:38:52,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +16: [2023-05-13 09:38:52,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +20: [2023-05-13 09:38:52,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:52,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +16: [2023-05-13 09:38:52,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +18: [2023-05-13 09:38:52,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:52,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:52,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:52,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:52,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:52,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 0: [2023-05-13 09:38:52,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 0: [2023-05-13 09:38:52,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +11: [2023-05-13 09:38:52,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +10: [2023-05-13 09:38:52,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:52,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:52,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:52,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:52,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:52,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +10: [2023-05-13 09:38:52,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:52,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:52,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:52,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:52,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 8: [2023-05-13 09:38:52,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +10: [2023-05-13 09:38:52,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 0: [2023-05-13 09:38:52,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +16: [2023-05-13 09:38:52,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:52,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:52,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:52,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:52,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:52,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:52,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:52,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:52,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +30: [2023-05-13 09:38:52,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:52,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +30: [2023-05-13 09:38:52,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:52,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:52,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:52,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:52,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:52,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 0: [2023-05-13 09:38:52,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:52,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:52,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:52,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:52,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:52,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:52,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +16: [2023-05-13 09:38:52,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +16: [2023-05-13 09:38:52,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 8: [2023-05-13 09:38:52,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 9: [2023-05-13 09:38:52,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 0: [2023-05-13 09:38:52,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 8: [2023-05-13 09:38:52,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 2: [2023-05-13 09:38:52,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 2: [2023-05-13 09:38:52,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 2: [2023-05-13 09:38:52,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:52,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:52,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:52,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:52,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 2: [2023-05-13 09:38:52,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:52,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:52,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:52,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:52,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:52,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:52,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:52,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:52,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:52,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +22: [2023-05-13 09:38:52,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:52,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:52,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:52,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +22: [2023-05-13 09:38:52,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:52,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +24: [2023-05-13 09:38:52,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:52,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:52,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:52,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:52,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:52,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:52,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:52,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +17: [2023-05-13 09:38:52,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:52,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:52,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:52,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:52,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:52,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:52,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:52,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:52,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:52,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:52,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:52,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:52,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:52,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:52,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +17: [2023-05-13 09:38:52,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:52,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:52,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +30: [2023-05-13 09:38:52,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:52,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:52,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:52,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 2: [2023-05-13 09:38:52,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:52,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:52,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +22: [2023-05-13 09:38:52,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:52,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:52,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:52,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:52,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:52,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:52,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:52,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:52,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:52,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:52,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +14: [2023-05-13 09:38:52,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:52,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:52,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +23: [2023-05-13 09:38:52,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +14: [2023-05-13 09:38:52,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +30: [2023-05-13 09:38:52,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:52,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:52,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:52,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +23: [2023-05-13 09:38:52,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 3: [2023-05-13 09:38:52,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +30: [2023-05-13 09:38:52,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:52,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:52,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:52,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:52,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +26: [2023-05-13 09:38:52,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:52,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +24: [2023-05-13 09:38:52,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +26: [2023-05-13 09:38:52,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:52,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:52,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +24: [2023-05-13 09:38:52,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:52,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +17: [2023-05-13 09:38:52,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:52,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:52,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:52,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:52,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:52,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +31: [2023-05-13 09:38:52,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:52,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +17: [2023-05-13 09:38:52,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +22: [2023-05-13 09:38:52,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:52,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:52,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +31: [2023-05-13 09:38:52,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +24: [2023-05-13 09:38:52,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +15: [2023-05-13 09:38:52,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:52,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +15: [2023-05-13 09:38:52,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:52,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +22: [2023-05-13 09:38:52,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:52,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:52,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +14: [2023-05-13 09:38:52,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:52,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:52,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:52,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:52,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +15: [2023-05-13 09:38:52,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:52,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +23: [2023-05-13 09:38:52,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +26: [2023-05-13 09:38:52,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:52,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +28: [2023-05-13 09:38:52,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +23: [2023-05-13 09:38:52,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:52,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +17: [2023-05-13 09:38:52,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:52,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:52,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:52,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:52,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +12: [2023-05-13 09:38:52,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:52,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:52,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:52,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:52,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:52,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:52,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:52,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. +12: [2023-05-13 09:38:52,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 5: [2023-05-13 09:38:52,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:52,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 8: [2023-05-13 09:38:52,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_16-model_01-model_states.pt. + 6: [2023-05-13 09:38:52,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:52,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:52,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:52,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +22: [2023-05-13 09:38:52,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:52,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:52,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +11: [2023-05-13 09:38:52,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:52,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:52,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:52,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +31: [2023-05-13 09:38:52,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +17: [2023-05-13 09:38:52,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:52,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +26: [2023-05-13 09:38:52,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:52,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:52,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +31: [2023-05-13 09:38:52,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:52,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +24: [2023-05-13 09:38:52,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +11: [2023-05-13 09:38:52,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:52,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +28: [2023-05-13 09:38:52,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +18: [2023-05-13 09:38:52,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +15: [2023-05-13 09:38:52,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:52,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +18: [2023-05-13 09:38:52,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:52,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +24: [2023-05-13 09:38:52,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:52,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:52,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +29: [2023-05-13 09:38:52,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +29: [2023-05-13 09:38:52,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:52,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:52,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +29: [2023-05-13 09:38:52,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:52,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:52,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 1: [2023-05-13 09:38:52,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:52,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:52,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +28: [2023-05-13 09:38:52,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:52,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:52,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:52,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:52,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:52,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 8: [2023-05-13 09:38:52,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +12: [2023-05-13 09:38:52,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +13: [2023-05-13 09:38:52,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +13: [2023-05-13 09:38:52,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +13: [2023-05-13 09:38:52,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +13: [2023-05-13 09:38:52,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:52,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:52,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:52,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 3: [2023-05-13 09:38:52,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +14: [2023-05-13 09:38:52,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:52,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:52,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +14: [2023-05-13 09:38:52,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +19: [2023-05-13 09:38:52,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +19: [2023-05-13 09:38:52,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:52,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +20: [2023-05-13 09:38:52,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +20: [2023-05-13 09:38:52,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:52,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:52,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +20: [2023-05-13 09:38:52,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:52,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +21: [2023-05-13 09:38:52,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +21: [2023-05-13 09:38:52,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +21: [2023-05-13 09:38:52,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +21: [2023-05-13 09:38:52,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +19: [2023-05-13 09:38:52,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 4: [2023-05-13 09:38:52,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:52,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +19: [2023-05-13 09:38:52,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +26: [2023-05-13 09:38:52,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +27: [2023-05-13 09:38:52,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +27: [2023-05-13 09:38:52,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:52,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:52,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 6: [2023-05-13 09:38:52,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:52,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:52,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:52,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt... +22: [2023-05-13 09:38:52,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:52,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:52,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +15: [2023-05-13 09:38:52,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:52,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +15: [2023-05-13 09:38:52,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +14: [2023-05-13 09:38:52,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +29: [2023-05-13 09:38:52,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:52,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:52,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:52,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:52,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:52,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:52,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:52,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:52,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:52,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:52,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:52,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:52,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:52,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:52,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:52,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:52,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:52,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:52,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:52,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:52,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:52,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +23: [2023-05-13 09:38:52,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:52,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +23: [2023-05-13 09:38:52,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +23: [2023-05-13 09:38:52,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 4: [2023-05-13 09:38:52,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:52,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:52,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +26: [2023-05-13 09:38:52,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:52,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:52,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:52,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +26: [2023-05-13 09:38:52,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:52,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:52,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +13: [2023-05-13 09:38:52,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +14: [2023-05-13 09:38:52,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +13: [2023-05-13 09:38:52,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:52,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:52,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:52,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +20: [2023-05-13 09:38:52,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:52,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:52,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +19: [2023-05-13 09:38:52,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:52,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:52,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:52,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:52,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +27: [2023-05-13 09:38:52,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:52,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:52,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:52,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:52,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +12: [2023-05-13 09:38:52,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:52,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +27: [2023-05-13 09:38:52,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:52,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:52,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:52,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:52,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:52,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +17: [2023-05-13 09:38:52,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:52,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:52,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 5: [2023-05-13 09:38:52,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 5: [2023-05-13 09:38:52,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 6: [2023-05-13 09:38:52,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:52,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:52,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +31: [2023-05-13 09:38:52,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +22: [2023-05-13 09:38:52,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:52,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:52,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:52,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:52,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:52,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:52,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +15: [2023-05-13 09:38:52,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:52,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:52,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +22: [2023-05-13 09:38:52,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +31: [2023-05-13 09:38:52,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +15: [2023-05-13 09:38:52,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 8: [2023-05-13 09:38:52,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +31: [2023-05-13 09:38:52,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +11: [2023-05-13 09:38:52,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +22: [2023-05-13 09:38:52,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:52,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:52,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 8: [2023-05-13 09:38:52,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +10: [2023-05-13 09:38:52,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... + 7: [2023-05-13 09:38:52,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:52,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +12: [2023-05-13 09:38:52,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +22: [2023-05-13 09:38:52,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:52,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:52,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +18: [2023-05-13 09:38:52,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:52,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:52,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:52,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:52,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:52,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +12: [2023-05-13 09:38:52,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +21: [2023-05-13 09:38:52,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +21: [2023-05-13 09:38:52,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:52,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:52,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:52,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:52,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:52,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +20: [2023-05-13 09:38:52,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:52,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:52,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:52,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:52,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:52,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +19: [2023-05-13 09:38:52,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:52,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:52,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:52,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:52,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:52,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:52,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:52,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:52,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:52,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:52,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:52,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 1: [2023-05-13 09:38:52,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:52,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:52,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +18: [2023-05-13 09:38:52,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:52,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:52,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:52,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:52,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +23: [2023-05-13 09:38:52,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 4: [2023-05-13 09:38:52,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:52,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:52,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:52,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +19: [2023-05-13 09:38:52,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +20: [2023-05-13 09:38:52,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 4: [2023-05-13 09:38:52,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:52,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:52,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:52,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:52,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:52,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +20: [2023-05-13 09:38:52,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +31: [2023-05-13 09:38:52,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:52,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 4: [2023-05-13 09:38:52,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 1: [2023-05-13 09:38:52,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:52,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +28: [2023-05-13 09:38:52,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:52,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:52,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:52,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:52,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +29: [2023-05-13 09:38:52,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:52,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:52,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:52,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:52,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +13: [2023-05-13 09:38:52,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +27: [2023-05-13 09:38:52,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:52,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_00-model_states.pt. +29: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +27: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +29: [2023-05-13 09:38:52,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:52,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +20: [2023-05-13 09:38:52,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 7: [2023-05-13 09:38:52,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:52,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:52,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 7: [2023-05-13 09:38:52,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +19: [2023-05-13 09:38:52,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:52,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt... +15: [2023-05-13 09:38:52,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:52,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:52,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:52,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:52,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:52,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 1: [2023-05-13 09:38:52,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:52,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:52,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:52,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +13: [2023-05-13 09:38:52,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:52,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:52,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +21: [2023-05-13 09:38:52,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:52,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +13: [2023-05-13 09:38:52,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:52,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:52,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +17: [2023-05-13 09:38:52,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:52,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:52,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:52,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +16: [2023-05-13 09:38:52,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +16: [2023-05-13 09:38:52,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +13: [2023-05-13 09:38:52,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +16: [2023-05-13 09:38:52,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +13: [2023-05-13 09:38:52,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +13: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +14: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +25: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +15: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +30: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +30: [2023-05-13 09:38:52,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:52,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:52,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:52,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:52,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:52,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:52,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:52,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +10: [2023-05-13 09:38:52,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 7: [2023-05-13 09:38:52,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:52,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:52,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:52,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 1: [2023-05-13 09:38:52,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:52,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +14: [2023-05-13 09:38:52,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +13: [2023-05-13 09:38:52,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:52,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:52,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:52,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:52,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:52,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:52,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:52,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:52,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:52,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:52,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +19: [2023-05-13 09:38:52,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:52,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:52,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:52,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:52,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:52,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:52,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +19: [2023-05-13 09:38:52,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:52,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:52,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:52,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:52,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:52,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:52,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:52,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +17: [2023-05-13 09:38:52,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +21: [2023-05-13 09:38:52,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:52,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:52,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:52,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:52,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:52,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:52,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 7: [2023-05-13 09:38:52,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:52,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 1: [2023-05-13 09:38:52,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:52,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +11: [2023-05-13 09:38:52,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +10: [2023-05-13 09:38:52,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:52,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:52,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:52,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:52,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:52,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:52,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:52,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +21: [2023-05-13 09:38:52,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:52,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:52,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:52,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:52,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:52,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:52,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:52,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:52,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:52,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:52,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:52,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:52,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:52,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:52,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:52,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:52,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:52,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:52,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +20: [2023-05-13 09:38:52,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:52,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:52,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +31: [2023-05-13 09:38:52,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:52,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:52,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:52,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +29: [2023-05-13 09:38:52,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:52,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:52,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:52,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:52,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:52,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:52,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:52,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +16: [2023-05-13 09:38:52,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:52,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:52,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:52,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:52,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:52,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:52,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:52,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:52,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +11: [2023-05-13 09:38:52,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:52,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:52,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:52,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +27: [2023-05-13 09:38:52,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:52,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:52,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +16: [2023-05-13 09:38:52,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +14: [2023-05-13 09:38:52,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:52,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:52,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +16: [2023-05-13 09:38:52,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:52,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:52,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:52,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:52,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:52,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:52,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:52,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:52,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +11: [2023-05-13 09:38:52,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:52,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:52,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:52,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:52,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:52,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:52,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 0: [2023-05-13 09:38:52,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +11: [2023-05-13 09:38:52,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:52,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +10: [2023-05-13 09:38:52,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:52,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +14: [2023-05-13 09:38:52,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +11: [2023-05-13 09:38:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +10: [2023-05-13 09:38:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:52,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:52,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:52,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:52,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:52,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:52,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:52,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +24: [2023-05-13 09:38:52,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +24: [2023-05-13 09:38:52,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 2: [2023-05-13 09:38:52,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:52,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +25: [2023-05-13 09:38:52,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +25: [2023-05-13 09:38:52,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:52,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:52,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:52,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +25: [2023-05-13 09:38:52,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:52,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:52,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:52,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:52,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +25: [2023-05-13 09:38:52,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:52,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:52,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:52,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:52,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:52,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:52,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +18: [2023-05-13 09:38:52,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:52,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:52,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:52,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:52,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:52,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:52,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:52,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +24: [2023-05-13 09:38:52,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +27: [2023-05-13 09:38:52,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:52,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:52,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 0: [2023-05-13 09:38:52,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:52,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +23: [2023-05-13 09:38:52,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +27: [2023-05-13 09:38:52,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:52,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:52,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:52,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:52,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:52,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +27: [2023-05-13 09:38:52,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 2: [2023-05-13 09:38:52,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:52,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:52,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 2: [2023-05-13 09:38:52,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:52,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 3: [2023-05-13 09:38:52,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:52,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:52,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:52,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +14: [2023-05-13 09:38:52,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +14: [2023-05-13 09:38:52,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +30: [2023-05-13 09:38:52,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:52,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:52,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +23: [2023-05-13 09:38:52,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:52,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:52,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:52,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 6: [2023-05-13 09:38:52,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 3: [2023-05-13 09:38:52,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +23: [2023-05-13 09:38:52,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 8: [2023-05-13 09:38:52,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 8: [2023-05-13 09:38:52,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 8: [2023-05-13 09:38:52,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +30: [2023-05-13 09:38:52,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:52,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:52,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:52,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:52,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:52,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 0: [2023-05-13 09:38:52,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:52,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:52,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:52,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:52,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:52,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:52,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +24: [2023-05-13 09:38:52,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:52,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:52,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:52,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:52,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:52,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:52,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +24: [2023-05-13 09:38:52,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 6: [2023-05-13 09:38:52,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:52,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:52,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +18: [2023-05-13 09:38:52,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:52,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 5: [2023-05-13 09:38:52,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 5: [2023-05-13 09:38:52,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:52,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:52,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 5: [2023-05-13 09:38:52,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:52,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:52,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:52,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:52,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +12: [2023-05-13 09:38:52,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +31: [2023-05-13 09:38:52,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. + 9: [2023-05-13 09:38:52,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:52,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:52,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:52,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:52,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +28: [2023-05-13 09:38:52,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_17-model_01-model_states.pt. +23: [2023-05-13 09:38:52,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +23: [2023-05-13 09:38:52,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:52,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 9: [2023-05-13 09:38:52,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 9: [2023-05-13 09:38:52,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +30: [2023-05-13 09:38:52,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:52,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:52,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +23: [2023-05-13 09:38:52,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +31: [2023-05-13 09:38:52,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:52,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:52,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +31: [2023-05-13 09:38:52,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:52,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:52,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:52,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:52,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:52,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:52,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +14: [2023-05-13 09:38:52,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:52,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 9: [2023-05-13 09:38:52,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +14: [2023-05-13 09:38:52,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +28: [2023-05-13 09:38:52,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +14: [2023-05-13 09:38:52,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +14: [2023-05-13 09:38:52,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 6: [2023-05-13 09:38:52,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 6: [2023-05-13 09:38:52,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 9: [2023-05-13 09:38:52,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:52,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +12: [2023-05-13 09:38:52,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:52,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt... +30: [2023-05-13 09:38:52,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:52,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 6: [2023-05-13 09:38:52,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 6: [2023-05-13 09:38:52,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 8: [2023-05-13 09:38:52,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 6: [2023-05-13 09:38:52,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +30: [2023-05-13 09:38:52,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:52,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:52,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:52,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +31: [2023-05-13 09:38:52,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 8: [2023-05-13 09:38:52,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:52,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:52,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:52,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:52,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:52,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:52,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:52,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:52,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:52,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:52,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:52,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:52,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +22: [2023-05-13 09:38:52,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:52,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:52,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +31: [2023-05-13 09:38:52,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:52,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:52,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +31: [2023-05-13 09:38:52,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:52,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:52,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:52,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +28: [2023-05-13 09:38:52,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +28: [2023-05-13 09:38:52,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:52,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:52,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:52,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +15: [2023-05-13 09:38:52,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:52,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:52,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +22: [2023-05-13 09:38:52,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:52,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:52,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... + 5: [2023-05-13 09:38:52,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +12: [2023-05-13 09:38:52,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_00-model_states.pt. +12: [2023-05-13 09:38:52,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt... +15: [2023-05-13 09:38:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:52,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:52,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:52,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:52,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:52,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:52,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:52,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:52,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:52,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:52,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:52,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:52,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:52,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:52,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:52,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:52,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:52,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:52,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:52,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:52,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:52,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:52,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:52,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:52,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:52,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:52,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:52,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:52,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:52,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:52,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:52,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:52,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:52,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:52,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:52,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:52,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:52,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:52,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:52,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:52,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:52,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:52,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:52,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:52,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:52,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:52,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:52,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:52,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:52,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:52,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:52,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:52,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:52,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:52,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:52,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:52,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:52,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:52,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:52,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:52,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:52,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:52,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:52,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:52,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:52,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:52,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:52,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:52,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:52,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:52,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:52,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:52,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:52,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:52,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:52,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:52,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +27: [2023-05-13 09:38:52,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:52,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:52,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:52,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:52,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:52,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +17: [2023-05-13 09:38:52,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +17: [2023-05-13 09:38:52,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:52,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:52,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:52,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:52,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:52,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:52,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:52,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:52,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:52,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:52,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:52,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:52,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:52,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:52,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:52,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:52,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:52,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:53,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:53,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:53,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:53,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:53,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:53,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:53,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:53,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:53,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:53,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:53,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:53,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:53,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:53,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:53,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:53,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:53,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:53,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:53,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:53,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:53,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:53,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:53,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:53,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:53,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:53,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:53,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:53,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:53,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:53,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:53,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:53,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:53,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:53,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:53,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:53,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:53,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:53,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:53,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:53,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:53,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:53,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:53,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:53,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:53,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:53,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:53,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:53,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:53,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:53,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:53,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:53,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:53,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:53,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:53,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:53,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:53,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:53,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:53,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:53,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:53,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:53,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:53,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:53,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:53,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:53,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:53,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:53,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:53,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:53,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:53,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:53,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:53,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:53,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:53,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:53,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:53,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:53,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:53,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:53,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:53,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +21: [2023-05-13 09:38:53,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:53,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:53,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:53,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:53,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:53,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:53,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:53,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:53,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:53,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:53,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:53,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:53,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:53,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:53,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:53,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:53,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:53,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:53,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:53,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:53,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:53,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:53,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:53,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:53,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:53,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:53,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:53,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:53,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:53,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:53,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:53,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:53,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:53,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:53,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:53,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:53,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:53,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:53,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:53,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:53,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:53,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:53,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:53,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:53,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:53,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:53,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:53,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:53,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:53,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:53,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:53,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:53,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:53,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:53,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:53,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:53,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:53,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:53,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:53,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:53,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:53,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:53,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:53,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:53,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:53,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:53,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:53,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:53,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:53,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:53,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:53,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:53,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:53,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:53,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:53,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:53,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:53,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:53,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:53,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:53,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:53,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:53,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:53,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:53,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:53,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:53,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:53,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:53,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:53,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:53,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:53,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:53,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:53,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:53,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:53,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:53,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:53,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:53,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:53,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:53,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:53,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +26: [2023-05-13 09:38:53,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:53,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +26: [2023-05-13 09:38:53,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:53,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:53,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +12: [2023-05-13 09:38:53,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:53,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:53,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:53,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:53,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:53,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:53,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +19: [2023-05-13 09:38:53,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +19: [2023-05-13 09:38:53,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +23: [2023-05-13 09:38:53,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +29: [2023-05-13 09:38:53,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +29: [2023-05-13 09:38:53,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +31: [2023-05-13 09:38:53,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:53,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:53,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:53,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:53,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:53,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:53,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:53,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:53,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:53,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:53,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:53,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:53,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:53,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:53,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:53,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:53,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:53,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:53,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:53,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:53,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:53,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:53,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:53,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:53,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:53,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:53,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:53,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:53,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:53,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +31: [2023-05-13 09:38:53,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:53,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:53,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:53,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:53,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:53,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:53,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:53,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:53,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:53,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:53,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:53,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:53,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:53,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:53,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:53,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:53,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:53,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:53,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:53,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:53,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:53,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:53,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:53,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:53,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:53,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:53,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:53,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:53,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:53,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:53,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:53,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:53,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:53,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +19: [2023-05-13 09:38:53,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:53,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:53,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:53,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:53,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +19: [2023-05-13 09:38:53,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:53,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:53,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +19: [2023-05-13 09:38:53,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:53,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:53,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:53,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:53,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:53,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +19: [2023-05-13 09:38:53,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:53,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:53,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:53,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:53,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:53,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:53,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:53,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:53,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:53,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:53,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:53,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:53,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:53,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:53,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:53,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:53,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:53,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:53,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:53,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:53,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:53,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:53,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:53,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:53,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:53,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:53,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:53,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:53,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:53,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:53,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:53,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:53,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:53,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:53,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:53,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:53,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:53,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:53,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:53,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:53,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:53,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:53,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:53,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:53,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:53,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:53,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:53,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:53,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:53,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:53,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:53,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:53,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:53,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +10: [2023-05-13 09:38:53,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:53,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +10: [2023-05-13 09:38:53,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:53,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:53,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 4: [2023-05-13 09:38:53,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 4: [2023-05-13 09:38:53,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 4: [2023-05-13 09:38:53,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:53,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:53,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:53,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:53,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 6: [2023-05-13 09:38:53,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:53,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:53,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:53,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:53,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:53,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:53,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:53,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:53,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:53,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:53,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:53,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:53,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:53,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:53,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:53,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:53,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:53,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:53,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:53,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:53,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:53,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:53,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:53,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:53,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:53,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:53,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:53,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:53,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:53,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:53,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:53,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:53,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:53,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:53,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:53,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:53,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:53,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:53,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:53,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +21: [2023-05-13 09:38:53,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:53,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 4: [2023-05-13 09:38:53,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:53,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +27: [2023-05-13 09:38:53,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +27: [2023-05-13 09:38:53,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +27: [2023-05-13 09:38:53,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +18: [2023-05-13 09:38:53,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +18: [2023-05-13 09:38:53,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +18: [2023-05-13 09:38:53,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:53,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:53,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:53,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:53,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:53,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:53,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:53,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:53,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 4: [2023-05-13 09:38:53,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +21: [2023-05-13 09:38:53,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 7: [2023-05-13 09:38:53,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +10: [2023-05-13 09:38:53,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +10: [2023-05-13 09:38:53,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 7: [2023-05-13 09:38:53,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:53,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 4: [2023-05-13 09:38:53,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 4: [2023-05-13 09:38:53,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 0: [2023-05-13 09:38:53,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:53,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +10: [2023-05-13 09:38:53,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:53,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:53,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:53,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:53,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:53,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:53,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:53,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:53,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +11: [2023-05-13 09:38:53,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:53,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:53,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:53,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:53,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:53,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:53,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:53,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:53,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:53,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:53,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:53,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +21: [2023-05-13 09:38:53,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:53,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:53,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:53,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:53,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:53,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:53,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:53,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:53,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:53,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:53,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:53,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:53,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:53,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:53,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:53,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:53,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:53,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:53,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:53,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:53,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:53,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:53,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +16: [2023-05-13 09:38:53,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +16: [2023-05-13 09:38:53,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:53,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:53,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:53,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:53,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:53,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:53,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 1: [2023-05-13 09:38:53,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:53,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:53,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:53,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:53,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:53,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:53,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:53,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:53,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:53,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:53,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 7: [2023-05-13 09:38:53,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:53,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:53,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 7: [2023-05-13 09:38:53,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +21: [2023-05-13 09:38:53,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:53,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +13: [2023-05-13 09:38:53,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:53,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:53,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +16: [2023-05-13 09:38:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:53,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:53,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:53,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +10: [2023-05-13 09:38:53,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:53,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:53,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:53,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:53,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:53,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:53,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:53,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:53,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:53,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:53,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:53,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:53,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:53,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:53,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:53,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:53,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:53,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:53,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:53,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:53,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:53,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:53,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +13: [2023-05-13 09:38:53,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:53,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:53,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +27: [2023-05-13 09:38:53,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 8: [2023-05-13 09:38:53,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:53,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:53,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:53,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:53,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:53,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +13: [2023-05-13 09:38:53,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:53,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:53,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +19: [2023-05-13 09:38:53,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:53,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:53,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:53,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:53,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:53,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:53,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:53,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:53,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:53,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:53,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:53,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:53,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:53,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +28: [2023-05-13 09:38:53,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:53,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 1: [2023-05-13 09:38:53,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +28: [2023-05-13 09:38:53,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:53,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:53,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:53,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 1: [2023-05-13 09:38:53,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 1: [2023-05-13 09:38:53,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +21: [2023-05-13 09:38:53,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:53,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:53,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:53,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:53,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:53,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:53,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:53,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:53,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +16: [2023-05-13 09:38:53,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +13: [2023-05-13 09:38:53,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:53,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:53,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +13: [2023-05-13 09:38:53,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:53,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:53,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:53,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:53,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +16: [2023-05-13 09:38:53,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:53,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:53,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:53,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +18: [2023-05-13 09:38:53,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +18: [2023-05-13 09:38:53,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:53,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:53,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:53,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:53,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:53,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +16: [2023-05-13 09:38:53,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 0: [2023-05-13 09:38:53,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:53,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:53,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +10: [2023-05-13 09:38:53,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +10: [2023-05-13 09:38:53,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:53,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:53,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:53,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:53,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:53,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:53,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:53,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:53,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +28: [2023-05-13 09:38:53,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:53,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:53,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:53,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 7: [2023-05-13 09:38:53,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +20: [2023-05-13 09:38:53,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +20: [2023-05-13 09:38:53,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 0: [2023-05-13 09:38:53,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:53,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +20: [2023-05-13 09:38:53,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:53,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:53,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +10: [2023-05-13 09:38:53,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:53,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:53,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:53,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:53,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:53,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:53,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:53,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:53,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:53,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:53,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:53,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:53,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:53,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:53,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:53,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:53,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:53,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:53,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:53,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:53,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:53,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:53,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:53,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:53,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +25: [2023-05-13 09:38:53,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +19: [2023-05-13 09:38:53,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:53,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:53,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:53,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:53,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:53,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:53,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:53,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +31: [2023-05-13 09:38:53,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +20: [2023-05-13 09:38:53,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:53,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:53,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:53,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:53,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:53,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:53,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:53,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:53,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:53,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:53,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +31: [2023-05-13 09:38:53,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:53,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:53,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:53,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:53,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +26: [2023-05-13 09:38:53,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:53,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:53,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:53,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:53,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:53,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +26: [2023-05-13 09:38:53,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +30: [2023-05-13 09:38:53,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +30: [2023-05-13 09:38:53,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:53,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:53,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +30: [2023-05-13 09:38:53,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +30: [2023-05-13 09:38:53,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:53,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +18: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:53,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +23: [2023-05-13 09:38:53,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +23: [2023-05-13 09:38:53,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +23: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +12: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +12: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +12: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +15: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +30: [2023-05-13 09:38:53,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:53,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +27: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:53,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:53,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:53,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:53,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +27: [2023-05-13 09:38:53,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:53,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +27: [2023-05-13 09:38:53,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:53,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:53,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:53,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +22: [2023-05-13 09:38:53,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +22: [2023-05-13 09:38:53,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +22: [2023-05-13 09:38:53,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +31: [2023-05-13 09:38:53,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 3: [2023-05-13 09:38:53,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +31: [2023-05-13 09:38:53,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:53,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 3: [2023-05-13 09:38:53,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +20: [2023-05-13 09:38:53,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:53,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:53,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +20: [2023-05-13 09:38:53,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:53,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:53,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 6: [2023-05-13 09:38:53,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 6: [2023-05-13 09:38:53,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:53,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:53,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +20: [2023-05-13 09:38:53,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 8: [2023-05-13 09:38:53,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 6: [2023-05-13 09:38:53,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 8: [2023-05-13 09:38:53,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:53,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:53,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:53,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 5: [2023-05-13 09:38:53,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:53,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:53,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:53,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 2: [2023-05-13 09:38:53,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +23: [2023-05-13 09:38:53,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:53,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:53,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:53,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:53,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:53,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:53,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:53,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:53,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:53,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:53,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:53,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:53,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:53,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:53,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:53,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:53,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:53,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:53,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:53,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 9: [2023-05-13 09:38:53,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 9: [2023-05-13 09:38:53,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 3: [2023-05-13 09:38:53,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. + 3: [2023-05-13 09:38:53,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:53,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:53,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:53,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:53,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:53,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:53,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:53,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +20: [2023-05-13 09:38:53,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +12: [2023-05-13 09:38:53,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +25: [2023-05-13 09:38:53,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:53,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:53,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:53,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +15: [2023-05-13 09:38:53,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:53,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:53,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +29: [2023-05-13 09:38:53,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +26: [2023-05-13 09:38:53,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:53,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:53,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:53,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:53,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:53,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:53,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:53,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:53,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:53,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:53,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:53,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:53,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:53,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +25: [2023-05-13 09:38:53,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +25: [2023-05-13 09:38:53,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:53,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +31: [2023-05-13 09:38:53,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:53,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:53,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 9: [2023-05-13 09:38:53,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +23: [2023-05-13 09:38:53,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:53,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:53,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:53,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:53,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:53,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:53,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:53,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +26: [2023-05-13 09:38:53,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +31: [2023-05-13 09:38:53,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +26: [2023-05-13 09:38:53,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:53,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:53,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +12: [2023-05-13 09:38:53,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:53,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:53,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:53,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:53,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +26: [2023-05-13 09:38:53,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:53,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:53,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +23: [2023-05-13 09:38:53,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:53,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +23: [2023-05-13 09:38:53,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:53,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:53,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +15: [2023-05-13 09:38:53,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +23: [2023-05-13 09:38:53,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +23: [2023-05-13 09:38:53,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +23: [2023-05-13 09:38:53,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:53,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 6: [2023-05-13 09:38:53,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:53,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +15: [2023-05-13 09:38:53,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +15: [2023-05-13 09:38:53,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +27: [2023-05-13 09:38:53,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +27: [2023-05-13 09:38:53,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:53,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +15: [2023-05-13 09:38:53,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +15: [2023-05-13 09:38:53,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:53,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:53,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:53,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 3: [2023-05-13 09:38:53,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:53,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 8: [2023-05-13 09:38:53,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +19: [2023-05-13 09:38:53,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +19: [2023-05-13 09:38:53,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +19: [2023-05-13 09:38:53,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:53,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 8: [2023-05-13 09:38:53,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +19: [2023-05-13 09:38:53,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:53,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:53,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:53,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:53,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:53,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:53,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:53,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 9: [2023-05-13 09:38:53,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:53,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:53,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:53,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +22: [2023-05-13 09:38:53,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:53,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +29: [2023-05-13 09:38:53,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +29: [2023-05-13 09:38:53,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +12: [2023-05-13 09:38:53,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +12: [2023-05-13 09:38:53,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 8: [2023-05-13 09:38:53,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 8: [2023-05-13 09:38:53,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:53,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:53,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:53,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 5: [2023-05-13 09:38:53,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:53,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:53,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +12: [2023-05-13 09:38:53,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 9: [2023-05-13 09:38:53,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 9: [2023-05-13 09:38:53,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +19: [2023-05-13 09:38:53,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +22: [2023-05-13 09:38:53,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +28: [2023-05-13 09:38:53,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:53,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:53,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +28: [2023-05-13 09:38:53,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +22: [2023-05-13 09:38:53,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:53,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:53,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +24: [2023-05-13 09:38:53,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +24: [2023-05-13 09:38:53,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +24: [2023-05-13 09:38:53,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:53,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:53,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:53,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +14: [2023-05-13 09:38:53,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +22: [2023-05-13 09:38:53,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:53,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +22: [2023-05-13 09:38:53,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:53,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:53,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:53,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:53,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:53,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:53,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +14: [2023-05-13 09:38:53,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +28: [2023-05-13 09:38:53,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:53,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +19: [2023-05-13 09:38:53,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:53,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +24: [2023-05-13 09:38:53,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:53,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:53,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:53,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:53,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:53,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +11: [2023-05-13 09:38:53,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +11: [2023-05-13 09:38:53,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +11: [2023-05-13 09:38:53,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_18-model_01-model_states.pt. +19: [2023-05-13 09:38:53,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:53,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:53,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +19: [2023-05-13 09:38:53,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +19: [2023-05-13 09:38:53,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:53,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:53,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:53,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:53,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:53,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:53,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +11: [2023-05-13 09:38:53,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt... +17: [2023-05-13 09:38:53,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +28: [2023-05-13 09:38:53,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:53,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:53,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +21: [2023-05-13 09:38:53,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +21: [2023-05-13 09:38:53,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +21: [2023-05-13 09:38:53,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +21: [2023-05-13 09:38:53,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:53,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:53,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:53,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:53,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:53,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:53,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:53,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +24: [2023-05-13 09:38:53,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +14: [2023-05-13 09:38:53,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +28: [2023-05-13 09:38:53,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +28: [2023-05-13 09:38:53,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:53,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +14: [2023-05-13 09:38:53,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 4: [2023-05-13 09:38:53,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:53,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +28: [2023-05-13 09:38:53,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +24: [2023-05-13 09:38:53,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:53,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... + 4: [2023-05-13 09:38:53,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:53,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +24: [2023-05-13 09:38:53,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +21: [2023-05-13 09:38:53,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:53,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +24: [2023-05-13 09:38:53,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +11: [2023-05-13 09:38:53,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:53,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:53,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +17: [2023-05-13 09:38:53,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:53,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +17: [2023-05-13 09:38:53,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +17: [2023-05-13 09:38:53,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:53,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +11: [2023-05-13 09:38:53,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:53,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_00-model_states.pt. +11: [2023-05-13 09:38:53,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +11: [2023-05-13 09:38:53,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt... +17: [2023-05-13 09:38:53,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:53,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:53,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:53,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:53,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:53,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:53,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:53,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:53,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:53,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +10: [2023-05-13 09:38:53,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +10: [2023-05-13 09:38:53,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +10: [2023-05-13 09:38:53,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:53,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +17: [2023-05-13 09:38:53,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:53,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:53,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:53,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:53,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:53,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:53,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:53,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:53,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:53,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:53,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:53,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +21: [2023-05-13 09:38:53,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:53,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:53,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:53,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:53,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +21: [2023-05-13 09:38:53,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:53,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:53,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:53,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:53,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 4: [2023-05-13 09:38:53,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:53,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:53,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:53,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:53,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:53,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:53,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:53,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 4: [2023-05-13 09:38:53,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 1: [2023-05-13 09:38:53,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 1: [2023-05-13 09:38:53,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 1: [2023-05-13 09:38:53,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:53,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:53,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:53,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:53,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:53,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:53,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:53,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:53,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:53,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:53,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:53,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:53,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:53,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:53,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:53,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:53,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:53,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:53,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:53,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:53,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:53,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:53,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:53,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:53,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:53,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:53,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:53,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:53,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:53,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:53,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:53,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:53,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +17: [2023-05-13 09:38:53,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:53,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:53,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:53,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:53,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:53,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:53,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +10: [2023-05-13 09:38:53,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 1: [2023-05-13 09:38:53,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:53,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:53,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:53,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:53,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:53,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:53,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +21: [2023-05-13 09:38:53,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:53,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:53,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:53,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:53,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:53,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:53,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:53,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:53,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 8: [2023-05-13 09:38:53,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:53,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +17: [2023-05-13 09:38:53,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 7: [2023-05-13 09:38:53,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:53,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:53,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +13: [2023-05-13 09:38:53,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:53,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:53,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:53,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:53,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +26: [2023-05-13 09:38:53,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:53,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:53,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:53,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:53,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:53,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +20: [2023-05-13 09:38:53,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:53,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:53,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:53,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:53,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +29: [2023-05-13 09:38:53,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 4: [2023-05-13 09:38:53,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:53,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 6: [2023-05-13 09:38:53,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:53,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:53,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:53,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:53,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:53,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:53,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:53,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:53,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:53,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +21: [2023-05-13 09:38:53,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:53,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:53,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:53,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:53,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:53,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:53,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:53,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:53,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:53,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:53,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:53,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:53,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:53,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:53,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:53,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:53,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:53,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:53,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:53,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:53,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:53,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:53,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:53,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:53,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:53,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:53,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:53,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:53,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:53,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:53,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:53,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:53,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:53,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:53,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:53,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:53,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:53,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:53,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:53,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:53,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:53,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:53,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:53,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:53,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +21: [2023-05-13 09:38:53,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:53,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:53,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:53,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:53,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:53,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:53,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:53,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:53,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:53,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:53,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:53,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:53,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:53,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +13: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:53,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +30: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +30: [2023-05-13 09:38:53,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +30: [2023-05-13 09:38:53,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:53,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:53,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:53,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:53,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:53,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:53,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 7: [2023-05-13 09:38:53,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +12: [2023-05-13 09:38:53,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 0: [2023-05-13 09:38:53,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:53,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:53,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 6: [2023-05-13 09:38:53,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:53,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:53,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:53,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:53,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:53,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:53,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:53,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:53,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:53,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:53,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:53,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:53,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:53,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:53,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 0: [2023-05-13 09:38:53,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:53,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:53,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:53,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +18: [2023-05-13 09:38:53,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +18: [2023-05-13 09:38:53,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:53,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:53,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:53,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:53,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:53,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:53,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:53,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:53,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:53,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:53,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:53,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:53,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:53,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:53,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:53,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:53,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:53,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:53,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:53,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:53,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:53,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:53,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:53,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:53,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:53,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:53,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:53,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:53,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:53,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:53,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:53,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:53,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:53,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:53,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:53,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:53,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:53,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:53,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:53,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:53,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:53,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:53,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:53,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:53,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +12: [2023-05-13 09:38:53,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:53,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:53,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:53,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 0: [2023-05-13 09:38:53,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:53,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +10: [2023-05-13 09:38:53,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:53,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:53,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:53,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:53,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:53,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:53,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:53,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:53,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:53,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:53,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:53,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:53,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:53,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:53,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:53,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:53,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:53,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:53,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:53,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:53,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:53,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:53,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:53,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:53,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:53,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:53,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:53,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:53,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:53,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:53,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 1: [2023-05-13 09:38:53,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 1: [2023-05-13 09:38:53,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +25: [2023-05-13 09:38:53,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:53,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:53,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:53,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:53,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:53,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +18: [2023-05-13 09:38:53,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:53,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:53,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:53,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:53,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:53,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:53,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:53,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:53,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:53,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:53,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:53,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +18: [2023-05-13 09:38:53,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:53,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:53,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:53,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:53,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:53,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +23: [2023-05-13 09:38:53,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +23: [2023-05-13 09:38:53,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +31: [2023-05-13 09:38:53,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:53,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:53,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:53,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:53,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:53,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:53,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +31: [2023-05-13 09:38:53,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:53,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +27: [2023-05-13 09:38:53,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +12: [2023-05-13 09:38:53,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:53,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +27: [2023-05-13 09:38:53,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:53,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:53,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:53,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:53,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:53,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:53,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 7: [2023-05-13 09:38:53,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:53,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:53,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:53,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:53,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:53,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:53,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:53,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:53,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:53,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:53,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:53,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:53,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:53,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:53,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:53,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:53,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 7: [2023-05-13 09:38:53,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:53,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:53,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +25: [2023-05-13 09:38:53,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:53,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:53,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:53,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:53,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +23: [2023-05-13 09:38:53,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +18: [2023-05-13 09:38:53,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +18: [2023-05-13 09:38:53,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +18: [2023-05-13 09:38:53,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +18: [2023-05-13 09:38:53,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +18: [2023-05-13 09:38:53,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:53,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:53,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:53,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:53,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:53,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:53,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:53,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:53,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:53,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:53,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:53,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:53,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:53,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:53,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:53,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:53,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:53,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +10: [2023-05-13 09:38:53,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:53,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:53,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:53,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:53,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:53,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:53,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 1: [2023-05-13 09:38:53,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 1: [2023-05-13 09:38:53,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:53,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:53,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +20: [2023-05-13 09:38:53,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:53,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:53,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:53,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +23: [2023-05-13 09:38:53,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:53,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:53,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:53,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:53,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:53,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:53,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:53,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:53,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +26: [2023-05-13 09:38:53,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +26: [2023-05-13 09:38:53,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +23: [2023-05-13 09:38:53,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:53,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +26: [2023-05-13 09:38:53,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +23: [2023-05-13 09:38:53,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +19: [2023-05-13 09:38:53,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:53,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:53,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:53,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +27: [2023-05-13 09:38:53,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:53,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +23: [2023-05-13 09:38:53,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:53,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:53,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 4: [2023-05-13 09:38:53,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:53,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:53,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +27: [2023-05-13 09:38:53,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:53,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:53,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:53,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +27: [2023-05-13 09:38:53,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +27: [2023-05-13 09:38:53,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:53,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:53,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:53,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:53,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:53,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:53,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:53,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:53,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:53,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:53,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:53,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:53,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 5: [2023-05-13 09:38:53,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 5: [2023-05-13 09:38:53,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:53,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:53,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:53,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:53,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:53,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 4: [2023-05-13 09:38:53,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 3: [2023-05-13 09:38:53,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 3: [2023-05-13 09:38:53,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 3: [2023-05-13 09:38:53,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:53,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:53,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:53,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:53,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:53,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:53,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:53,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:53,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:53,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:53,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:53,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:53,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:53,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +22: [2023-05-13 09:38:53,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:53,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +20: [2023-05-13 09:38:53,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 3: [2023-05-13 09:38:53,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:53,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:53,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +20: [2023-05-13 09:38:53,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +29: [2023-05-13 09:38:53,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:53,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 9: [2023-05-13 09:38:53,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 9: [2023-05-13 09:38:53,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +20: [2023-05-13 09:38:53,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +20: [2023-05-13 09:38:53,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +14: [2023-05-13 09:38:53,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 9: [2023-05-13 09:38:53,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +14: [2023-05-13 09:38:53,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:53,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:53,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:53,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +17: [2023-05-13 09:38:53,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +17: [2023-05-13 09:38:53,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +17: [2023-05-13 09:38:53,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +14: [2023-05-13 09:38:53,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +14: [2023-05-13 09:38:53,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:53,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:53,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:53,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +22: [2023-05-13 09:38:53,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:53,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:53,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +10: [2023-05-13 09:38:53,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +10: [2023-05-13 09:38:53,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +10: [2023-05-13 09:38:53,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +19: [2023-05-13 09:38:53,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:53,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +19: [2023-05-13 09:38:53,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:53,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +29: [2023-05-13 09:38:53,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:53,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:53,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:53,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:53,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +29: [2023-05-13 09:38:53,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +19: [2023-05-13 09:38:53,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:53,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:53,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +17: [2023-05-13 09:38:53,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:53,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:53,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:53,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +16: [2023-05-13 09:38:53,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +16: [2023-05-13 09:38:53,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +16: [2023-05-13 09:38:53,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +10: [2023-05-13 09:38:53,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:53,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:53,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:53,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:53,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:53,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:53,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +22: [2023-05-13 09:38:53,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:53,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:53,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:53,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 3: [2023-05-13 09:38:53,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 3: [2023-05-13 09:38:53,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +22: [2023-05-13 09:38:53,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +14: [2023-05-13 09:38:53,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:53,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:53,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:53,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 5: [2023-05-13 09:38:53,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:53,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +21: [2023-05-13 09:38:53,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:53,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:53,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:53,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +21: [2023-05-13 09:38:53,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +21: [2023-05-13 09:38:53,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 3: [2023-05-13 09:38:53,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:53,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 5: [2023-05-13 09:38:53,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 5: [2023-05-13 09:38:53,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:53,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +14: [2023-05-13 09:38:53,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +14: [2023-05-13 09:38:53,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 3: [2023-05-13 09:38:53,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:53,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 5: [2023-05-13 09:38:53,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:53,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +14: [2023-05-13 09:38:53,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +14: [2023-05-13 09:38:53,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +17: [2023-05-13 09:38:53,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:53,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:53,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:53,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +16: [2023-05-13 09:38:53,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +17: [2023-05-13 09:38:53,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +11: [2023-05-13 09:38:53,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +22: [2023-05-13 09:38:53,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +29: [2023-05-13 09:38:53,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:53,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:53,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:53,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +22: [2023-05-13 09:38:53,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +11: [2023-05-13 09:38:53,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:53,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +29: [2023-05-13 09:38:53,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:53,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +15: [2023-05-13 09:38:53,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:53,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +24: [2023-05-13 09:38:53,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:53,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:53,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:53,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 2: [2023-05-13 09:38:53,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:53,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:53,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +24: [2023-05-13 09:38:53,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:53,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:53,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:53,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:53,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:53,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +21: [2023-05-13 09:38:53,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +21: [2023-05-13 09:38:53,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:53,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:53,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:53,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:53,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:53,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:53,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:53,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:53,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +10: [2023-05-13 09:38:53,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:53,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:53,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:53,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +19: [2023-05-13 09:38:53,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:53,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:53,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:53,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:53,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:53,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 9: [2023-05-13 09:38:53,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +10: [2023-05-13 09:38:53,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:53,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:53,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:53,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:53,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +19: [2023-05-13 09:38:53,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:53,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:53,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:53,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +15: [2023-05-13 09:38:53,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:53,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +10: [2023-05-13 09:38:53,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:53,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:53,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +11: [2023-05-13 09:38:53,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:53,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:53,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:53,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:53,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:53,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:53,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:53,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:53,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +16: [2023-05-13 09:38:53,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +11: [2023-05-13 09:38:53,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +21: [2023-05-13 09:38:53,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:53,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:53,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +16: [2023-05-13 09:38:53,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +16: [2023-05-13 09:38:53,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 6: [2023-05-13 09:38:53,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 6: [2023-05-13 09:38:53,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 6: [2023-05-13 09:38:53,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 6: [2023-05-13 09:38:53,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +11: [2023-05-13 09:38:53,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:53,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:53,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:53,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:53,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +24: [2023-05-13 09:38:53,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:53,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:53,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:53,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:53,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:53,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:53,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:53,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:53,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +21: [2023-05-13 09:38:53,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:53,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 2: [2023-05-13 09:38:53,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +15: [2023-05-13 09:38:53,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +21: [2023-05-13 09:38:53,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:53,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +12: [2023-05-13 09:38:53,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +12: [2023-05-13 09:38:53,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +12: [2023-05-13 09:38:53,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +30: [2023-05-13 09:38:53,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:53,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:53,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:53,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:53,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +30: [2023-05-13 09:38:53,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:53,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +15: [2023-05-13 09:38:53,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:53,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:53,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 6: [2023-05-13 09:38:53,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:53,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +19: [2023-05-13 09:38:53,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:53,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:53,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +24: [2023-05-13 09:38:53,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:53,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:53,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:53,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:53,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:53,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:53,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +15: [2023-05-13 09:38:53,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:53,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:53,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +15: [2023-05-13 09:38:53,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:53,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:53,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:53,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:53,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:53,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:53,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:53,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:53,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:53,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:53,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. + 8: [2023-05-13 09:38:53,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +28: [2023-05-13 09:38:53,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:53,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:53,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:53,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:53,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:53,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:53,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:53,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +31: [2023-05-13 09:38:53,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:53,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:53,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:53,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 1: [2023-05-13 09:38:53,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:53,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:53,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +31: [2023-05-13 09:38:53,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:53,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 0: [2023-05-13 09:38:53,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 0: [2023-05-13 09:38:53,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 0: [2023-05-13 09:38:53,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 0: [2023-05-13 09:38:53,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +31: [2023-05-13 09:38:53,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:53,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:53,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:53,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:53,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:53,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:53,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +24: [2023-05-13 09:38:53,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:53,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +13: [2023-05-13 09:38:53,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +21: [2023-05-13 09:38:53,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:53,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:53,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:53,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:53,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +21: [2023-05-13 09:38:54,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:54,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:54,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 0: [2023-05-13 09:38:54,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +25: [2023-05-13 09:38:54,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 8: [2023-05-13 09:38:54,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:54,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:54,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +25: [2023-05-13 09:38:54,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:54,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:54,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 4: [2023-05-13 09:38:54,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:54,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:54,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:54,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:54,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +12: [2023-05-13 09:38:54,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:54,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:54,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:54,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +28: [2023-05-13 09:38:54,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:54,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:54,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +28: [2023-05-13 09:38:54,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +13: [2023-05-13 09:38:54,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:54,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:54,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:54,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:54,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:54,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:54,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +26: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +18: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +18: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +23: [2023-05-13 09:38:54,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:54,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:54,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +23: [2023-05-13 09:38:54,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:54,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +28: [2023-05-13 09:38:54,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:54,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:54,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:54,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:54,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 1: [2023-05-13 09:38:54,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +12: [2023-05-13 09:38:54,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +26: [2023-05-13 09:38:54,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:54,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:54,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:54,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +12: [2023-05-13 09:38:54,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +30: [2023-05-13 09:38:54,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:54,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 8: [2023-05-13 09:38:54,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +30: [2023-05-13 09:38:54,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:54,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +12: [2023-05-13 09:38:54,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 8: [2023-05-13 09:38:54,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:54,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:54,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +31: [2023-05-13 09:38:54,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:54,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:54,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:54,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:54,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:54,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:54,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:54,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:54,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:54,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:54,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:54,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +26: [2023-05-13 09:38:54,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:54,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:54,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:54,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +25: [2023-05-13 09:38:54,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:54,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:54,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:54,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:54,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +31: [2023-05-13 09:38:54,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:54,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +27: [2023-05-13 09:38:54,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +27: [2023-05-13 09:38:54,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:54,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +21: [2023-05-13 09:38:54,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:54,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:54,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:54,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:54,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:54,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:54,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:54,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:54,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:54,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +17: [2023-05-13 09:38:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:54,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:54,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:54,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:54,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:54,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:54,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +29: [2023-05-13 09:38:54,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:54,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:54,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:54,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +24: [2023-05-13 09:38:54,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +24: [2023-05-13 09:38:54,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +13: [2023-05-13 09:38:54,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:54,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_19-model_01-model_states.pt. +29: [2023-05-13 09:38:54,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:54,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:54,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:54,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:54,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:54,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:54,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:54,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:54,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:54,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:54,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:54,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:54,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:54,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:54,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:54,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:54,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +18: [2023-05-13 09:38:54,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:54,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:54,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:54,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:54,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:54,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:54,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +18: [2023-05-13 09:38:54,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:54,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:54,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:54,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +24: [2023-05-13 09:38:54,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt... +30: [2023-05-13 09:38:54,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +30: [2023-05-13 09:38:54,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +18: [2023-05-13 09:38:54,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +18: [2023-05-13 09:38:54,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:54,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:54,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +22: [2023-05-13 09:38:54,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +22: [2023-05-13 09:38:54,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 0: [2023-05-13 09:38:54,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +17: [2023-05-13 09:38:54,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:54,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:54,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:54,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:54,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:54,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:54,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:54,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:54,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:54,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:54,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:54,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:54,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:54,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:54,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:54,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:54,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:54,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:54,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:54,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:54,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:54,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:54,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:54,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:54,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:54,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:54,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:54,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:54,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:54,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:54,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:54,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:54,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:54,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:54,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +16: [2023-05-13 09:38:54,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:54,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:54,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:54,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:54,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:54,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +17: [2023-05-13 09:38:54,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:54,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:54,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:54,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:54,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:54,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:54,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:54,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:54,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:54,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:54,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +23: [2023-05-13 09:38:54,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +27: [2023-05-13 09:38:54,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 9: [2023-05-13 09:38:54,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:54,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:54,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:54,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:54,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:54,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:54,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:54,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:54,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:54,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:54,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:54,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:54,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:54,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:54,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:54,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:54,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:54,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:54,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:54,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +29: [2023-05-13 09:38:54,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:54,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:54,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:54,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:54,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:54,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:54,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:54,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:54,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:54,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:54,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:54,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:54,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:54,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +23: [2023-05-13 09:38:54,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:54,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:54,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:54,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:54,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:54,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:54,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. +24: [2023-05-13 09:38:54,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:54,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:54,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +24: [2023-05-13 09:38:54,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... +22: [2023-05-13 09:38:54,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +24: [2023-05-13 09:38:54,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:54,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:54,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:54,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:54,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:54,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:54,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:54,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:54,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:54,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:54,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:54,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:54,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:54,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:54,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:54,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:54,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:54,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:54,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:54,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:54,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 4: [2023-05-13 09:38:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:54,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:54,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:54,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:54,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:54,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:54,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:54,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:54,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:54,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:54,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:54,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:54,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:54,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:54,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:54,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:54,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:54,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:54,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:54,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:54,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:54,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:54,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:54,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:54,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:54,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +20: [2023-05-13 09:38:54,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:54,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:54,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:54,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:54,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:54,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:54,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:54,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:54,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:54,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:54,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:54,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:54,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:54,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:54,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:54,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:54,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:54,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:54,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:54,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:54,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:54,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:54,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:54,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:54,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:54,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:54,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:54,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:54,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:54,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:54,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:54,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:54,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:54,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:54,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:54,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:54,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:54,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +13: [2023-05-13 09:38:54,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:54,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:54,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:54,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:54,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:54,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:54,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:54,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:54,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:54,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:54,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +20: [2023-05-13 09:38:54,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:54,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:54,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:54,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:54,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:54,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:54,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:54,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:54,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:54,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:54,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:54,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:54,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:54,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +21: [2023-05-13 09:38:54,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +25: [2023-05-13 09:38:54,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:54,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +21: [2023-05-13 09:38:54,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +28: [2023-05-13 09:38:54,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:54,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:54,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:54,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:54,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:54,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:54,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:54,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +31: [2023-05-13 09:38:54,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:54,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:54,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:54,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:54,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:54,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 5: [2023-05-13 09:38:54,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 5: [2023-05-13 09:38:54,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 5: [2023-05-13 09:38:54,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 1: [2023-05-13 09:38:54,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 1: [2023-05-13 09:38:54,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +18: [2023-05-13 09:38:54,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:54,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 1: [2023-05-13 09:38:54,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +22: [2023-05-13 09:38:54,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:54,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:54,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:54,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:54,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:54,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:54,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:54,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:54,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +10: [2023-05-13 09:38:54,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:54,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:54,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +10: [2023-05-13 09:38:54,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +21: [2023-05-13 09:38:54,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:54,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:54,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:54,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:54,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:54,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:54,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:54,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:54,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:54,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:54,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:54,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:54,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:54,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:54,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:54,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +22: [2023-05-13 09:38:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +29: [2023-05-13 09:38:54,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:54,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:54,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:54,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:54,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:54,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +20: [2023-05-13 09:38:54,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +20: [2023-05-13 09:38:54,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:54,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:54,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +25: [2023-05-13 09:38:54,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:54,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:54,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +17: [2023-05-13 09:38:54,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +17: [2023-05-13 09:38:54,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +17: [2023-05-13 09:38:54,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:54,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:54,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +25: [2023-05-13 09:38:54,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:54,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +10: [2023-05-13 09:38:54,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:54,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:54,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:54,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:54,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:54,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:54,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:54,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:54,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:54,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:54,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:54,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:54,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:54,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +19: [2023-05-13 09:38:54,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +29: [2023-05-13 09:38:54,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:54,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:54,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +11: [2023-05-13 09:38:54,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:54,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:54,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:54,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:54,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:54,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:54,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +11: [2023-05-13 09:38:54,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:54,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +25: [2023-05-13 09:38:54,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:54,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +14: [2023-05-13 09:38:54,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:54,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:54,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +14: [2023-05-13 09:38:54,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +21: [2023-05-13 09:38:54,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:54,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:54,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:54,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 2: [2023-05-13 09:38:54,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +22: [2023-05-13 09:38:54,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:54,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:54,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +25: [2023-05-13 09:38:54,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:54,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 2: [2023-05-13 09:38:54,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +31: [2023-05-13 09:38:54,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:54,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:54,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +29: [2023-05-13 09:38:54,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:54,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:54,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:54,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:54,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:54,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +29: [2023-05-13 09:38:54,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +21: [2023-05-13 09:38:54,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +29: [2023-05-13 09:38:54,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 2: [2023-05-13 09:38:54,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:54,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +19: [2023-05-13 09:38:54,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:54,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 1: [2023-05-13 09:38:54,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +10: [2023-05-13 09:38:54,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 2: [2023-05-13 09:38:54,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:54,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:54,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:54,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:54,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:54,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:54,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +28: [2023-05-13 09:38:54,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:54,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 2: [2023-05-13 09:38:54,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +10: [2023-05-13 09:38:54,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 1: [2023-05-13 09:38:54,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +10: [2023-05-13 09:38:54,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +26: [2023-05-13 09:38:54,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:54,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:54,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:54,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:54,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:54,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:54,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:54,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:54,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 1: [2023-05-13 09:38:54,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:54,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:54,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:54,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +19: [2023-05-13 09:38:54,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:54,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:54,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:54,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:54,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:54,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:54,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:54,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:54,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:54,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:54,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:54,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:54,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +19: [2023-05-13 09:38:54,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:54,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +17: [2023-05-13 09:38:54,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 6: [2023-05-13 09:38:54,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 6: [2023-05-13 09:38:54,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 6: [2023-05-13 09:38:54,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +14: [2023-05-13 09:38:54,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:54,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:54,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:54,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:54,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +14: [2023-05-13 09:38:54,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:54,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:54,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:54,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:54,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:54,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +11: [2023-05-13 09:38:54,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +14: [2023-05-13 09:38:54,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +11: [2023-05-13 09:38:54,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:54,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +14: [2023-05-13 09:38:54,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +11: [2023-05-13 09:38:54,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:54,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:54,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:54,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:54,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +14: [2023-05-13 09:38:54,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 2: [2023-05-13 09:38:54,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:54,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:54,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 2: [2023-05-13 09:38:54,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 2: [2023-05-13 09:38:54,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +26: [2023-05-13 09:38:54,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:54,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:54,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:54,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +28: [2023-05-13 09:38:54,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 3: [2023-05-13 09:38:54,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 3: [2023-05-13 09:38:54,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:54,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 3: [2023-05-13 09:38:54,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +11: [2023-05-13 09:38:54,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:54,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +15: [2023-05-13 09:38:54,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +15: [2023-05-13 09:38:54,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +11: [2023-05-13 09:38:54,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +15: [2023-05-13 09:38:54,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 4: [2023-05-13 09:38:54,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:54,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:54,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +11: [2023-05-13 09:38:54,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:54,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +31: [2023-05-13 09:38:54,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +31: [2023-05-13 09:38:54,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 1: [2023-05-13 09:38:54,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:54,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:54,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:54,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:54,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +25: [2023-05-13 09:38:54,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +25: [2023-05-13 09:38:54,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:54,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +25: [2023-05-13 09:38:54,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +28: [2023-05-13 09:38:54,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:54,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:54,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:54,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:54,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:54,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:54,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +30: [2023-05-13 09:38:54,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:54,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:54,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +28: [2023-05-13 09:38:54,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:54,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:54,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +16: [2023-05-13 09:38:54,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +30: [2023-05-13 09:38:54,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:54,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:54,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:54,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:54,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:54,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:54,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +18: [2023-05-13 09:38:54,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +18: [2023-05-13 09:38:54,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +18: [2023-05-13 09:38:54,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +18: [2023-05-13 09:38:54,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:54,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +31: [2023-05-13 09:38:54,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:54,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:54,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +16: [2023-05-13 09:38:54,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:54,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:54,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:54,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:54,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:54,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:54,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:54,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:54,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:54,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:54,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:54,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:54,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:54,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +23: [2023-05-13 09:38:54,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +13: [2023-05-13 09:38:54,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 1: [2023-05-13 09:38:54,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:54,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:54,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:54,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +25: [2023-05-13 09:38:54,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:54,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:54,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +14: [2023-05-13 09:38:54,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:54,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +14: [2023-05-13 09:38:54,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:54,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:54,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +26: [2023-05-13 09:38:54,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:54,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:54,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:54,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 0: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +16: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 0: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:54,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:54,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:54,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +16: [2023-05-13 09:38:54,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:54,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:54,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:54,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:54,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:54,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:54,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:54,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:54,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +18: [2023-05-13 09:38:54,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:54,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +30: [2023-05-13 09:38:54,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +30: [2023-05-13 09:38:54,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:54,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +30: [2023-05-13 09:38:54,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 6: [2023-05-13 09:38:54,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +13: [2023-05-13 09:38:54,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +11: [2023-05-13 09:38:54,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:54,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:54,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:54,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:54,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:54,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 3: [2023-05-13 09:38:54,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +11: [2023-05-13 09:38:54,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:54,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:54,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:54,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:54,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:54,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:54,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:54,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:54,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +13: [2023-05-13 09:38:54,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:54,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:54,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:54,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 0: [2023-05-13 09:38:54,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:54,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 3: [2023-05-13 09:38:54,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:54,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:54,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:54,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:54,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:54,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:54,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +12: [2023-05-13 09:38:54,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:54,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:54,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +23: [2023-05-13 09:38:54,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:54,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:54,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. + 8: [2023-05-13 09:38:54,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +30: [2023-05-13 09:38:54,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:54,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +12: [2023-05-13 09:38:54,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:54,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:54,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +28: [2023-05-13 09:38:54,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:54,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:54,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:54,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:54,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:54,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:54,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:54,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:54,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:54,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:54,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:54,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:54,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:54,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:54,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +22: [2023-05-13 09:38:54,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +26: [2023-05-13 09:38:54,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +22: [2023-05-13 09:38:54,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +14: [2023-05-13 09:38:54,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:54,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:54,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +15: [2023-05-13 09:38:54,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +26: [2023-05-13 09:38:54,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:54,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:54,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:54,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +25: [2023-05-13 09:38:54,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:54,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +27: [2023-05-13 09:38:54,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +27: [2023-05-13 09:38:54,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +30: [2023-05-13 09:38:54,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:54,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +30: [2023-05-13 09:38:54,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:54,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:54,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +31: [2023-05-13 09:38:54,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 3: [2023-05-13 09:38:54,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:54,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:54,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:54,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +14: [2023-05-13 09:38:54,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +30: [2023-05-13 09:38:54,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:54,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +26: [2023-05-13 09:38:54,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:54,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +24: [2023-05-13 09:38:54,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +16: [2023-05-13 09:38:54,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:54,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:54,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_20-model_01-model_states.pt. +26: [2023-05-13 09:38:54,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:54,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 3: [2023-05-13 09:38:54,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +18: [2023-05-13 09:38:54,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:54,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +18: [2023-05-13 09:38:54,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +26: [2023-05-13 09:38:54,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:54,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:54,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:54,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:54,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:54,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:54,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:54,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +18: [2023-05-13 09:38:54,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +18: [2023-05-13 09:38:54,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +18: [2023-05-13 09:38:54,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +31: [2023-05-13 09:38:54,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:54,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:54,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +18: [2023-05-13 09:38:54,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:54,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:54,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:54,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:54,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:54,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:54,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:54,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:54,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:54,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:54,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:54,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:54,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:54,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:54,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:54,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:54,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:54,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +27: [2023-05-13 09:38:54,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:54,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:54,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:54,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:54,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:54,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:54,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:54,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:54,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:54,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... +13: [2023-05-13 09:38:54,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:54,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:54,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:54,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +13: [2023-05-13 09:38:54,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:54,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:54,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:54,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:54,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:54,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:54,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:54,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:54,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:54,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +11: [2023-05-13 09:38:54,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:54,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:54,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:54,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:54,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:54,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:54,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:54,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:54,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:54,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:54,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:54,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:54,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:54,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +31: [2023-05-13 09:38:54,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:54,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:54,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +25: [2023-05-13 09:38:54,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:54,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:54,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:54,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +30: [2023-05-13 09:38:54,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:54,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:54,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +22: [2023-05-13 09:38:54,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:54,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +22: [2023-05-13 09:38:54,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 8: [2023-05-13 09:38:54,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +12: [2023-05-13 09:38:54,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:54,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:54,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:54,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +13: [2023-05-13 09:38:54,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:54,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:54,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:54,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +30: [2023-05-13 09:38:54,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +22: [2023-05-13 09:38:54,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +22: [2023-05-13 09:38:54,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:54,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:54,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:54,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:54,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:54,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +27: [2023-05-13 09:38:54,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +25: [2023-05-13 09:38:54,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:54,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:54,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +30: [2023-05-13 09:38:54,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +11: [2023-05-13 09:38:54,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:54,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +27: [2023-05-13 09:38:54,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:54,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 0: [2023-05-13 09:38:54,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:54,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +24: [2023-05-13 09:38:54,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. +31: [2023-05-13 09:38:54,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +23: [2023-05-13 09:38:54,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:54,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:54,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... + 8: [2023-05-13 09:38:54,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:54,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:54,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:54,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:54,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:54,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:54,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:54,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:54,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:54,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:54,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:54,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:54,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:54,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:54,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:54,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt... +15: [2023-05-13 09:38:54,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:54,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:54,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:54,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +31: [2023-05-13 09:38:54,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:54,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:54,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:54,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:54,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:54,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:54,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:54,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:54,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:54,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:54,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:54,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:54,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:54,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:54,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:54,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:54,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:54,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:54,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:54,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:54,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:54,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:54,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +15: [2023-05-13 09:38:54,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:54,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:54,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:54,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:54,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:54,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +23: [2023-05-13 09:38:54,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:54,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:54,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +23: [2023-05-13 09:38:54,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:54,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:54,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:54,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:54,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:54,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:54,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:54,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:54,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +23: [2023-05-13 09:38:54,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +27: [2023-05-13 09:38:54,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:54,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:54,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:54,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:54,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:54,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:54,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:54,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:54,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:54,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:54,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:54,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:54,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:54,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:54,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:54,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:54,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 5: [2023-05-13 09:38:54,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 5: [2023-05-13 09:38:54,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:54,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +17: [2023-05-13 09:38:54,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:54,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:54,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:54,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:54,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:54,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:54,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:54,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:54,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:54,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:54,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:54,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:54,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:54,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:54,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:54,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +27: [2023-05-13 09:38:54,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:54,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:54,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +14: [2023-05-13 09:38:54,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:54,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +21: [2023-05-13 09:38:54,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:54,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +20: [2023-05-13 09:38:54,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:54,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +21: [2023-05-13 09:38:54,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:54,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:54,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:54,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +27: [2023-05-13 09:38:54,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:54,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:54,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:54,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:54,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:54,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:54,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:54,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:54,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +21: [2023-05-13 09:38:54,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:54,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:54,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:54,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:54,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:54,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:54,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +14: [2023-05-13 09:38:54,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:54,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:54,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:54,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:54,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:54,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +27: [2023-05-13 09:38:54,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:54,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:54,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:54,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:54,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:54,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:54,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:54,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:54,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:54,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:54,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:54,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:54,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:54,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:54,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:54,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:54,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:54,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:54,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:54,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:54,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:54,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:54,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:54,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:54,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:54,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:54,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:54,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:54,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:54,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:54,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:54,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:54,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:54,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:54,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:54,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +17: [2023-05-13 09:38:54,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:54,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:54,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 2: [2023-05-13 09:38:54,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:54,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:54,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +14: [2023-05-13 09:38:54,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +15: [2023-05-13 09:38:54,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:54,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:54,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +14: [2023-05-13 09:38:54,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +15: [2023-05-13 09:38:54,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 1: [2023-05-13 09:38:54,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 1: [2023-05-13 09:38:54,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +14: [2023-05-13 09:38:54,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +15: [2023-05-13 09:38:54,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 1: [2023-05-13 09:38:54,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 1: [2023-05-13 09:38:54,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:54,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +16: [2023-05-13 09:38:54,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:54,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:54,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:54,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +12: [2023-05-13 09:38:54,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 2: [2023-05-13 09:38:54,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:54,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 6: [2023-05-13 09:38:54,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +18: [2023-05-13 09:38:54,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:54,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 6: [2023-05-13 09:38:54,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +16: [2023-05-13 09:38:54,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +16: [2023-05-13 09:38:54,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:54,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:54,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:54,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:54,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:54,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +19: [2023-05-13 09:38:54,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:54,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:54,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:54,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +19: [2023-05-13 09:38:54,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:54,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:54,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:54,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +16: [2023-05-13 09:38:54,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 3: [2023-05-13 09:38:54,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 3: [2023-05-13 09:38:54,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:54,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:54,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:54,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 3: [2023-05-13 09:38:54,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:54,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:54,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:54,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:54,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:54,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:54,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:54,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +11: [2023-05-13 09:38:54,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +18: [2023-05-13 09:38:54,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:54,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:54,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:54,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:54,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:54,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:54,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:54,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:54,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +24: [2023-05-13 09:38:54,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:54,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:54,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:54,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:54,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:54,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:54,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:54,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:54,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +18: [2023-05-13 09:38:54,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:54,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +15: [2023-05-13 09:38:54,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:54,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:54,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:54,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +18: [2023-05-13 09:38:54,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:54,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:54,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:54,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:54,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:54,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:54,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 1: [2023-05-13 09:38:54,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +15: [2023-05-13 09:38:54,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +28: [2023-05-13 09:38:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +21: [2023-05-13 09:38:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:54,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 4: [2023-05-13 09:38:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:54,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:54,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:54,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 1: [2023-05-13 09:38:54,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 1: [2023-05-13 09:38:54,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 1: [2023-05-13 09:38:54,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:54,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +15: [2023-05-13 09:38:54,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:54,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 1: [2023-05-13 09:38:54,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +15: [2023-05-13 09:38:54,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:54,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +12: [2023-05-13 09:38:54,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +28: [2023-05-13 09:38:54,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +28: [2023-05-13 09:38:54,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:54,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:54,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:54,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +24: [2023-05-13 09:38:54,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +12: [2023-05-13 09:38:54,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +12: [2023-05-13 09:38:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +12: [2023-05-13 09:38:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +24: [2023-05-13 09:38:54,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. + 8: [2023-05-13 09:38:54,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +10: [2023-05-13 09:38:54,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_21-model_01-model_states.pt. +11: [2023-05-13 09:38:54,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +12: [2023-05-13 09:38:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 3: [2023-05-13 09:38:54,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +11: [2023-05-13 09:38:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +10: [2023-05-13 09:38:54,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +21: [2023-05-13 09:38:54,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:54,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:54,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:54,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +11: [2023-05-13 09:38:54,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:54,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +10: [2023-05-13 09:38:54,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +10: [2023-05-13 09:38:54,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +11: [2023-05-13 09:38:54,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +10: [2023-05-13 09:38:54,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:54,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +21: [2023-05-13 09:38:54,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +10: [2023-05-13 09:38:54,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:54,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +21: [2023-05-13 09:38:54,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +11: [2023-05-13 09:38:54,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +11: [2023-05-13 09:38:54,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:54,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +21: [2023-05-13 09:38:54,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +11: [2023-05-13 09:38:54,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:54,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:54,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:54,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +24: [2023-05-13 09:38:54,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:54,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +20: [2023-05-13 09:38:54,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:54,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:54,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:54,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:54,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 8: [2023-05-13 09:38:54,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:54,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:54,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +24: [2023-05-13 09:38:54,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt... +22: [2023-05-13 09:38:54,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:54,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:54,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:54,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:54,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:54,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:54,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +22: [2023-05-13 09:38:54,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 9: [2023-05-13 09:38:54,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +19: [2023-05-13 09:38:54,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:54,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:54,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:54,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +28: [2023-05-13 09:38:54,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +28: [2023-05-13 09:38:54,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:54,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +28: [2023-05-13 09:38:54,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 4: [2023-05-13 09:38:54,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 4: [2023-05-13 09:38:54,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 4: [2023-05-13 09:38:54,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +28: [2023-05-13 09:38:54,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:54,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:54,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:54,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:54,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:54,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +24: [2023-05-13 09:38:54,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... + 8: [2023-05-13 09:38:54,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +24: [2023-05-13 09:38:54,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +20: [2023-05-13 09:38:54,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +20: [2023-05-13 09:38:54,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:54,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +22: [2023-05-13 09:38:54,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:54,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +22: [2023-05-13 09:38:54,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:54,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +24: [2023-05-13 09:38:54,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +24: [2023-05-13 09:38:54,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_00-model_states.pt. +22: [2023-05-13 09:38:54,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:54,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +24: [2023-05-13 09:38:54,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt... +22: [2023-05-13 09:38:54,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:54,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:54,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:54,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:54,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:54,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 1: [2023-05-13 09:38:54,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:54,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:54,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:54,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:54,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:54,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:54,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +26: [2023-05-13 09:38:54,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:54,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:54,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:54,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:54,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +17: [2023-05-13 09:38:54,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +17: [2023-05-13 09:38:54,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +20: [2023-05-13 09:38:54,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:54,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +29: [2023-05-13 09:38:54,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:54,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:54,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +31: [2023-05-13 09:38:54,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:54,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:54,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:54,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +31: [2023-05-13 09:38:54,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +31: [2023-05-13 09:38:54,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:54,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:54,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +26: [2023-05-13 09:38:54,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +26: [2023-05-13 09:38:54,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +19: [2023-05-13 09:38:54,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +20: [2023-05-13 09:38:54,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:54,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:54,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:54,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:54,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:54,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +18: [2023-05-13 09:38:54,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +18: [2023-05-13 09:38:54,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +18: [2023-05-13 09:38:54,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 4: [2023-05-13 09:38:54,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:54,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +27: [2023-05-13 09:38:54,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +27: [2023-05-13 09:38:54,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +27: [2023-05-13 09:38:54,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +17: [2023-05-13 09:38:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 4: [2023-05-13 09:38:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +30: [2023-05-13 09:38:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +30: [2023-05-13 09:38:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +30: [2023-05-13 09:38:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +14: [2023-05-13 09:38:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +29: [2023-05-13 09:38:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +31: [2023-05-13 09:38:54,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:54,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:54,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:54,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:54,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +18: [2023-05-13 09:38:54,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 4: [2023-05-13 09:38:54,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +27: [2023-05-13 09:38:54,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:54,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:54,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:54,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:54,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +30: [2023-05-13 09:38:54,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:54,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:54,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:54,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:54,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:54,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:54,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:54,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:54,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:54,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:54,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:54,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:54,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:54,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:54,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:54,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:54,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:54,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:54,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:54,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:54,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:54,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:54,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:54,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:54,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:54,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:54,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:54,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:54,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:54,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:54,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +17: [2023-05-13 09:38:54,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:54,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:54,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:54,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:54,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:54,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:54,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:54,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:54,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:54,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:54,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:54,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 1: [2023-05-13 09:38:54,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:54,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:54,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:54,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:54,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:54,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:54,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:54,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:54,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:54,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:54,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:54,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:54,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +31: [2023-05-13 09:38:54,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:54,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:54,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:54,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:54,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +18: [2023-05-13 09:38:54,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:54,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:54,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:54,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:54,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:54,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +18: [2023-05-13 09:38:54,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +18: [2023-05-13 09:38:54,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +18: [2023-05-13 09:38:54,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +27: [2023-05-13 09:38:54,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:54,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +27: [2023-05-13 09:38:54,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +18: [2023-05-13 09:38:54,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:54,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:54,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:54,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:54,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:54,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:54,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:54,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:54,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +27: [2023-05-13 09:38:54,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:54,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:54,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:54,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:54,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:54,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:54,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:54,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:54,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:54,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:54,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:54,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:54,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:54,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:54,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:54,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:54,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:54,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:54,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:54,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:54,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:54,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:54,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:54,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:54,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:54,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:54,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:54,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:54,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:54,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:54,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:54,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:54,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:54,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:54,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:54,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:54,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:54,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:54,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:54,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:54,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:54,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:54,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:54,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:54,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:54,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:54,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:54,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:54,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:54,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:54,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:54,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +22: [2023-05-13 09:38:54,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:54,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:54,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:54,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:54,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:54,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +25: [2023-05-13 09:38:54,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:54,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:54,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:54,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:54,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 5: [2023-05-13 09:38:54,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 5: [2023-05-13 09:38:54,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:54,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:54,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:54,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:54,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:54,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:54,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:54,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:54,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:54,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:54,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:54,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +10: [2023-05-13 09:38:54,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:54,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:54,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 2: [2023-05-13 09:38:54,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 2: [2023-05-13 09:38:54,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:54,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +23: [2023-05-13 09:38:54,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +23: [2023-05-13 09:38:54,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +23: [2023-05-13 09:38:54,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:54,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:54,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 6: [2023-05-13 09:38:54,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:54,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:54,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:54,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:54,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:54,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:54,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:54,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:54,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:54,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:54,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:54,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:54,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:54,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:54,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:54,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:54,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:54,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +10: [2023-05-13 09:38:54,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:54,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:54,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:54,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:54,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:54,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:54,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:54,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:54,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:54,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:54,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:54,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:54,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:54,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:54,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:54,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:54,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:54,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:54,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:54,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:54,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:54,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:54,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:54,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:54,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:54,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:54,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:54,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:54,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:54,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:54,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:54,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:54,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:54,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:54,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +23: [2023-05-13 09:38:54,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:54,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:54,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:54,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:54,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:54,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:54,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +23: [2023-05-13 09:38:54,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:54,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:54,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:54,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +23: [2023-05-13 09:38:54,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 0: [2023-05-13 09:38:54,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:54,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:54,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:54,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:54,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:54,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:54,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:54,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 5: [2023-05-13 09:38:54,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:54,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:54,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:54,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:54,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:54,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:54,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:54,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:54,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:54,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:54,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +25: [2023-05-13 09:38:54,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:54,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:54,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +25: [2023-05-13 09:38:54,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:54,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:54,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:54,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +12: [2023-05-13 09:38:54,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:54,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 8: [2023-05-13 09:38:54,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 8: [2023-05-13 09:38:54,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:54,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 8: [2023-05-13 09:38:54,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:54,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +20: [2023-05-13 09:38:54,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +20: [2023-05-13 09:38:54,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:54,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:54,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:54,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:54,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:54,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +13: [2023-05-13 09:38:54,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +13: [2023-05-13 09:38:54,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:54,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:54,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:54,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:54,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:54,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +11: [2023-05-13 09:38:54,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:54,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:54,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:54,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:54,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:54,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:54,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:54,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:54,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:54,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:54,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:54,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:54,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:54,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:54,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:54,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:54,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +12: [2023-05-13 09:38:54,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:54,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:54,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +20: [2023-05-13 09:38:54,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:54,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:54,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 9: [2023-05-13 09:38:54,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:54,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:54,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:54,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +12: [2023-05-13 09:38:54,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:54,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 2: [2023-05-13 09:38:54,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:54,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +13: [2023-05-13 09:38:54,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 2: [2023-05-13 09:38:54,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:54,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:54,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:54,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:54,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:54,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +13: [2023-05-13 09:38:54,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:54,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:54,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:54,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:54,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:54,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:54,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 5: [2023-05-13 09:38:54,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:54,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:55,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:55,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:55,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:55,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:55,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:55,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:55,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:55,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:55,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:55,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:55,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +21: [2023-05-13 09:38:55,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 2: [2023-05-13 09:38:55,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:55,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:55,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 3: [2023-05-13 09:38:55,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +14: [2023-05-13 09:38:55,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +14: [2023-05-13 09:38:55,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:55,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +14: [2023-05-13 09:38:55,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 8: [2023-05-13 09:38:55,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:55,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 2: [2023-05-13 09:38:55,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:55,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:55,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 8: [2023-05-13 09:38:55,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +12: [2023-05-13 09:38:55,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:55,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:55,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:55,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:55,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:55,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:55,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:55,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:55,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:55,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:55,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:55,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 0: [2023-05-13 09:38:55,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:55,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +25: [2023-05-13 09:38:55,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +20: [2023-05-13 09:38:55,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:55,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:55,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:55,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:55,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:55,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 5: [2023-05-13 09:38:55,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:55,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:55,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:55,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:55,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:55,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:55,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +12: [2023-05-13 09:38:55,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:55,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:55,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:55,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:55,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:55,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:55,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +24: [2023-05-13 09:38:55,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +14: [2023-05-13 09:38:55,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:55,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +11: [2023-05-13 09:38:55,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:55,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +24: [2023-05-13 09:38:55,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 3: [2023-05-13 09:38:55,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +14: [2023-05-13 09:38:55,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:55,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +15: [2023-05-13 09:38:55,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 3: [2023-05-13 09:38:55,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +25: [2023-05-13 09:38:55,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +12: [2023-05-13 09:38:55,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:55,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +15: [2023-05-13 09:38:55,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:55,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +20: [2023-05-13 09:38:55,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:55,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:55,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 9: [2023-05-13 09:38:55,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +22: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:55,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 9: [2023-05-13 09:38:55,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:55,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:55,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +15: [2023-05-13 09:38:55,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:55,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:55,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:55,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:55,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:55,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +22: [2023-05-13 09:38:55,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:55,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +17: [2023-05-13 09:38:55,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:55,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:55,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +17: [2023-05-13 09:38:55,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +22: [2023-05-13 09:38:55,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:55,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:55,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:55,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:55,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:55,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:55,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:55,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:55,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:55,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:55,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +26: [2023-05-13 09:38:55,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +10: [2023-05-13 09:38:55,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +10: [2023-05-13 09:38:55,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +10: [2023-05-13 09:38:55,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +10: [2023-05-13 09:38:55,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:55,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +21: [2023-05-13 09:38:55,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +30: [2023-05-13 09:38:55,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +26: [2023-05-13 09:38:55,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:55,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:55,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:55,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +29: [2023-05-13 09:38:55,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +29: [2023-05-13 09:38:55,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +21: [2023-05-13 09:38:55,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 0: [2023-05-13 09:38:55,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:55,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:55,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:55,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +24: [2023-05-13 09:38:55,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:55,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:55,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:55,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +17: [2023-05-13 09:38:55,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:55,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +11: [2023-05-13 09:38:55,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:55,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +24: [2023-05-13 09:38:55,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +26: [2023-05-13 09:38:55,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +26: [2023-05-13 09:38:55,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:55,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +11: [2023-05-13 09:38:55,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:55,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:55,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:55,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:55,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +10: [2023-05-13 09:38:55,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:55,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:55,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:55,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:55,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +10: [2023-05-13 09:38:55,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:55,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:55,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:55,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:55,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +29: [2023-05-13 09:38:55,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:55,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:55,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:55,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:55,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:55,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +14: [2023-05-13 09:38:55,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:55,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:55,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:55,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:55,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:55,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:55,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 3: [2023-05-13 09:38:55,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:55,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:55,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:55,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:55,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +14: [2023-05-13 09:38:55,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +29: [2023-05-13 09:38:55,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:55,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:55,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:55,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:55,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 3: [2023-05-13 09:38:55,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:55,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:55,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:55,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:55,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:55,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:55,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +16: [2023-05-13 09:38:55,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:55,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +28: [2023-05-13 09:38:55,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:55,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +30: [2023-05-13 09:38:55,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:55,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +22: [2023-05-13 09:38:55,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:55,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_22-model_01-model_states.pt. +28: [2023-05-13 09:38:55,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:55,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:55,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:55,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:55,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:55,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:55,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:55,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:55,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +22: [2023-05-13 09:38:55,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:55,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +30: [2023-05-13 09:38:55,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +15: [2023-05-13 09:38:55,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:55,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:55,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 8: [2023-05-13 09:38:55,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:55,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +10: [2023-05-13 09:38:55,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +10: [2023-05-13 09:38:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:55,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:55,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:55,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +22: [2023-05-13 09:38:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +27: [2023-05-13 09:38:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:55,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +17: [2023-05-13 09:38:55,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:55,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:55,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +18: [2023-05-13 09:38:55,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +16: [2023-05-13 09:38:55,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:55,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:55,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +19: [2023-05-13 09:38:55,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:55,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:55,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:55,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +19: [2023-05-13 09:38:55,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:55,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:55,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:55,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +19: [2023-05-13 09:38:55,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:55,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:55,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:55,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:55,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +24: [2023-05-13 09:38:55,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:55,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:55,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:55,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:55,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:55,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +28: [2023-05-13 09:38:55,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:55,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +10: [2023-05-13 09:38:55,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:55,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +10: [2023-05-13 09:38:55,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:55,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:55,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:55,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:55,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:55,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:55,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +10: [2023-05-13 09:38:55,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:55,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:55,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:55,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:55,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:55,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:55,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:55,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt... +15: [2023-05-13 09:38:55,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:55,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:55,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:55,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:55,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:55,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:55,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:55,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:55,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:55,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:55,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:55,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:55,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:55,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:55,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +29: [2023-05-13 09:38:55,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:55,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:55,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +17: [2023-05-13 09:38:55,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:55,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:55,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:55,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:55,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:55,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:55,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +27: [2023-05-13 09:38:55,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +27: [2023-05-13 09:38:55,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:55,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:55,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:55,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:55,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +31: [2023-05-13 09:38:55,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +31: [2023-05-13 09:38:55,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +31: [2023-05-13 09:38:55,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +16: [2023-05-13 09:38:55,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +16: [2023-05-13 09:38:55,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +26: [2023-05-13 09:38:55,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:55,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +16: [2023-05-13 09:38:55,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:55,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:55,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:55,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:55,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +26: [2023-05-13 09:38:55,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +19: [2023-05-13 09:38:55,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:55,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:55,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:55,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:55,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:55,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +28: [2023-05-13 09:38:55,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +27: [2023-05-13 09:38:55,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:55,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:55,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:55,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:55,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:55,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +17: [2023-05-13 09:38:55,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +17: [2023-05-13 09:38:55,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +17: [2023-05-13 09:38:55,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +31: [2023-05-13 09:38:55,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:55,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:55,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +19: [2023-05-13 09:38:55,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:55,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +17: [2023-05-13 09:38:55,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:55,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +21: [2023-05-13 09:38:55,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:55,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +31: [2023-05-13 09:38:55,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +21: [2023-05-13 09:38:55,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:55,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:55,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +26: [2023-05-13 09:38:55,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:55,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:55,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:55,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:55,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:55,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:55,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +29: [2023-05-13 09:38:55,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +30: [2023-05-13 09:38:55,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +18: [2023-05-13 09:38:55,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:55,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. +28: [2023-05-13 09:38:55,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... +18: [2023-05-13 09:38:55,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:55,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +18: [2023-05-13 09:38:55,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt... + 4: [2023-05-13 09:38:55,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:55,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:55,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:55,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +18: [2023-05-13 09:38:55,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:55,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:55,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:55,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +21: [2023-05-13 09:38:55,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:55,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:55,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:55,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:55,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:55,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:55,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:55,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:55,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:55,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:55,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:55,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:55,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:55,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:55,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:55,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:55,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +22: [2023-05-13 09:38:55,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:55,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:55,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:55,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +22: [2023-05-13 09:38:55,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +27: [2023-05-13 09:38:55,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:55,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:55,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:55,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:55,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:55,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:55,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +19: [2023-05-13 09:38:55,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +30: [2023-05-13 09:38:55,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:55,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:55,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:55,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +30: [2023-05-13 09:38:55,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:55,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:55,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +24: [2023-05-13 09:38:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +30: [2023-05-13 09:38:55,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +24: [2023-05-13 09:38:55,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:55,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:55,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +22: [2023-05-13 09:38:55,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:55,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:55,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:55,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:55,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:55,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:55,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:55,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:55,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:55,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:55,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:55,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +19: [2023-05-13 09:38:55,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:55,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:55,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:55,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:55,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:55,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:55,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:55,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:55,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:55,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:55,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:55,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:55,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:55,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:55,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:55,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:55,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:55,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:55,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:55,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:55,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:55,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +24: [2023-05-13 09:38:55,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:55,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +22: [2023-05-13 09:38:55,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:55,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:55,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:55,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:55,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +23: [2023-05-13 09:38:55,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +23: [2023-05-13 09:38:55,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +24: [2023-05-13 09:38:55,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:55,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:55,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:55,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:55,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:55,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:55,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:55,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:55,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:55,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +19: [2023-05-13 09:38:55,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +19: [2023-05-13 09:38:55,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:55,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:55,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:55,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:55,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:55,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:55,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:55,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:55,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:55,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:55,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:55,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:55,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 5: [2023-05-13 09:38:55,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 5: [2023-05-13 09:38:55,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:55,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 5: [2023-05-13 09:38:55,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:55,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:55,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 2: [2023-05-13 09:38:55,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +24: [2023-05-13 09:38:55,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:55,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:55,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:55,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:55,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:55,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:55,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:55,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:55,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 2: [2023-05-13 09:38:55,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:55,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:55,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:55,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 5: [2023-05-13 09:38:55,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 5: [2023-05-13 09:38:55,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +23: [2023-05-13 09:38:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:55,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +23: [2023-05-13 09:38:55,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:55,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:55,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:55,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:55,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:55,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:55,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 2: [2023-05-13 09:38:55,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:55,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 2: [2023-05-13 09:38:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 2: [2023-05-13 09:38:55,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 0: [2023-05-13 09:38:55,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 5: [2023-05-13 09:38:55,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:55,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 2: [2023-05-13 09:38:55,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 0: [2023-05-13 09:38:55,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 0: [2023-05-13 09:38:55,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 2: [2023-05-13 09:38:55,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 5: [2023-05-13 09:38:55,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:55,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 5: [2023-05-13 09:38:55,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +10: [2023-05-13 09:38:55,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +10: [2023-05-13 09:38:55,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 5: [2023-05-13 09:38:55,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 5: [2023-05-13 09:38:55,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 5: [2023-05-13 09:38:55,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +10: [2023-05-13 09:38:55,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +10: [2023-05-13 09:38:55,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 9: [2023-05-13 09:38:55,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:55,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:55,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:55,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:55,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:55,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:55,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:55,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:55,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:55,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:55,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:55,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:55,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:55,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:55,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:55,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:55,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:55,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +13: [2023-05-13 09:38:55,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:55,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:55,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:55,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:55,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:55,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:55,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +15: [2023-05-13 09:38:55,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:55,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:55,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:55,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:55,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:55,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +24: [2023-05-13 09:38:55,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +24: [2023-05-13 09:38:55,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +24: [2023-05-13 09:38:55,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +11: [2023-05-13 09:38:55,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +11: [2023-05-13 09:38:55,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +11: [2023-05-13 09:38:55,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +11: [2023-05-13 09:38:55,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:55,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 7: [2023-05-13 09:38:55,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 7: [2023-05-13 09:38:55,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 7: [2023-05-13 09:38:55,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:55,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:55,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +14: [2023-05-13 09:38:55,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +14: [2023-05-13 09:38:55,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +14: [2023-05-13 09:38:55,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:55,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:55,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:55,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:55,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +13: [2023-05-13 09:38:55,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +13: [2023-05-13 09:38:55,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +16: [2023-05-13 09:38:55,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +25: [2023-05-13 09:38:55,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:55,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:55,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:55,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:55,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:55,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:55,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:55,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:55,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 9: [2023-05-13 09:38:55,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:55,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:55,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:55,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +16: [2023-05-13 09:38:55,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:55,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:55,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:55,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:55,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:55,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:55,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +24: [2023-05-13 09:38:55,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:55,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 9: [2023-05-13 09:38:55,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:55,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 7: [2023-05-13 09:38:55,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:55,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +14: [2023-05-13 09:38:55,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:55,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 3: [2023-05-13 09:38:55,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +13: [2023-05-13 09:38:55,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:55,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 3: [2023-05-13 09:38:55,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:55,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:55,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:55,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:55,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:55,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +12: [2023-05-13 09:38:55,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +12: [2023-05-13 09:38:55,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +13: [2023-05-13 09:38:55,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:55,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:55,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:55,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +20: [2023-05-13 09:38:55,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +13: [2023-05-13 09:38:55,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:55,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +13: [2023-05-13 09:38:55,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:55,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +20: [2023-05-13 09:38:55,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:55,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:55,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +20: [2023-05-13 09:38:55,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +20: [2023-05-13 09:38:55,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:55,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:55,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:55,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:55,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +16: [2023-05-13 09:38:55,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +20: [2023-05-13 09:38:55,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +11: [2023-05-13 09:38:55,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:55,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:55,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:55,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:55,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:55,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +26: [2023-05-13 09:38:55,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +26: [2023-05-13 09:38:55,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +13: [2023-05-13 09:38:55,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:55,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +16: [2023-05-13 09:38:55,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:55,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:55,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 3: [2023-05-13 09:38:55,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +25: [2023-05-13 09:38:55,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:55,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +12: [2023-05-13 09:38:55,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:55,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +15: [2023-05-13 09:38:55,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:55,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:55,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:55,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +13: [2023-05-13 09:38:55,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +12: [2023-05-13 09:38:55,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +11: [2023-05-13 09:38:55,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:55,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +15: [2023-05-13 09:38:55,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +12: [2023-05-13 09:38:55,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +15: [2023-05-13 09:38:55,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +11: [2023-05-13 09:38:55,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:55,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:55,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:55,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:55,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +24: [2023-05-13 09:38:55,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:55,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +11: [2023-05-13 09:38:55,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:55,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:55,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:55,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:55,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +14: [2023-05-13 09:38:55,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +14: [2023-05-13 09:38:55,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +25: [2023-05-13 09:38:55,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +25: [2023-05-13 09:38:55,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:55,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:55,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:55,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:55,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +13: [2023-05-13 09:38:55,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:55,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:55,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:55,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:55,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +17: [2023-05-13 09:38:55,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:55,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:55,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:55,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:55,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:55,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:55,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 4: [2023-05-13 09:38:55,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:55,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +26: [2023-05-13 09:38:55,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +26: [2023-05-13 09:38:55,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:55,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:55,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +31: [2023-05-13 09:38:55,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 3: [2023-05-13 09:38:55,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:55,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +16: [2023-05-13 09:38:55,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:55,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:55,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:55,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 3: [2023-05-13 09:38:55,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:55,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:55,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:55,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +31: [2023-05-13 09:38:55,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 3: [2023-05-13 09:38:55,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +12: [2023-05-13 09:38:55,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +13: [2023-05-13 09:38:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:55,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 3: [2023-05-13 09:38:55,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +13: [2023-05-13 09:38:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:55,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:55,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +13: [2023-05-13 09:38:55,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +13: [2023-05-13 09:38:55,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 8: [2023-05-13 09:38:55,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:55,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:55,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:55,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +12: [2023-05-13 09:38:55,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:55,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:55,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +16: [2023-05-13 09:38:55,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +28: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +28: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +28: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +17: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +27: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. +28: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_23-model_01-model_states.pt. + 8: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +17: [2023-05-13 09:38:55,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:55,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:55,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +12: [2023-05-13 09:38:55,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +16: [2023-05-13 09:38:55,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:55,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:55,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +29: [2023-05-13 09:38:55,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +31: [2023-05-13 09:38:55,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:55,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:55,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:55,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +26: [2023-05-13 09:38:55,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:55,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:55,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:55,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +26: [2023-05-13 09:38:55,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:55,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:55,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:55,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +27: [2023-05-13 09:38:55,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:55,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:55,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... +28: [2023-05-13 09:38:55,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:55,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:55,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:55,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:55,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:55,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 4: [2023-05-13 09:38:55,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:55,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 4: [2023-05-13 09:38:55,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:55,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:55,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:55,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:55,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:55,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +22: [2023-05-13 09:38:55,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +22: [2023-05-13 09:38:55,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +22: [2023-05-13 09:38:55,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:55,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:55,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:55,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +17: [2023-05-13 09:38:55,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:55,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:55,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +19: [2023-05-13 09:38:55,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +27: [2023-05-13 09:38:55,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:55,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:55,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:55,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:55,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:55,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +19: [2023-05-13 09:38:55,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +24: [2023-05-13 09:38:55,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:55,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:55,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:55,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:55,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +28: [2023-05-13 09:38:55,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +24: [2023-05-13 09:38:55,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:55,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:55,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +22: [2023-05-13 09:38:55,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:55,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:55,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:55,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +27: [2023-05-13 09:38:55,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:55,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:55,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:55,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:55,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_00-model_states.pt. +27: [2023-05-13 09:38:55,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:55,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt... +30: [2023-05-13 09:38:55,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:55,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:55,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:55,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:55,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:55,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:55,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:55,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:55,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:55,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:55,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:55,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:55,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:55,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:55,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:55,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:55,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:55,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:55,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:55,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:55,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +21: [2023-05-13 09:38:55,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +21: [2023-05-13 09:38:55,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +21: [2023-05-13 09:38:55,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +22: [2023-05-13 09:38:55,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:55,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:55,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:55,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 4: [2023-05-13 09:38:55,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:55,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:55,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:55,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:55,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:55,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:55,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:55,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:55,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:55,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:55,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:55,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:55,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:55,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:55,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:55,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:55,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:55,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +22: [2023-05-13 09:38:55,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:55,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:55,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:55,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:55,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:55,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:55,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:55,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +17: [2023-05-13 09:38:55,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:55,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:55,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:55,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:55,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:55,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:55,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:55,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:55,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:55,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:55,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:55,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:55,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:55,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:55,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:55,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:55,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:55,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:55,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:55,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:55,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:55,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:55,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +18: [2023-05-13 09:38:55,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +18: [2023-05-13 09:38:55,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +18: [2023-05-13 09:38:55,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +23: [2023-05-13 09:38:55,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +22: [2023-05-13 09:38:55,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:55,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:55,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:55,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:55,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:55,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:55,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:55,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:55,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:55,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:55,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:55,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:55,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:55,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:55,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:55,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +21: [2023-05-13 09:38:55,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:55,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:55,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:55,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:55,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:55,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:55,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:55,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:55,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:55,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:55,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +21: [2023-05-13 09:38:55,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:55,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:55,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:55,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:55,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:55,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:55,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:55,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +26: [2023-05-13 09:38:55,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:55,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:55,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:55,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:55,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:55,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:55,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:55,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:55,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:55,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:55,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:55,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:55,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:55,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:55,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:55,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:55,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:55,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:55,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +19: [2023-05-13 09:38:55,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +19: [2023-05-13 09:38:55,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +19: [2023-05-13 09:38:55,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:55,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:55,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:55,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:55,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:55,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:55,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:55,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:55,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:55,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:55,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:55,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:55,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:55,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 2: [2023-05-13 09:38:55,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:55,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:55,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 5: [2023-05-13 09:38:55,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 5: [2023-05-13 09:38:55,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 5: [2023-05-13 09:38:55,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:55,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:55,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:55,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:55,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:55,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +26: [2023-05-13 09:38:55,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:55,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:55,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:55,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:55,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:55,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:55,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:55,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:55,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:55,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:55,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:55,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:55,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:55,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:55,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:55,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:55,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +29: [2023-05-13 09:38:55,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:55,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:55,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:55,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:55,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:55,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:55,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:55,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:55,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:55,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:55,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:55,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:55,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +19: [2023-05-13 09:38:55,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:55,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:55,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:55,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:55,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:55,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:55,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:55,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:55,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:55,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:55,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:55,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:55,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:55,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:55,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +15: [2023-05-13 09:38:55,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:55,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:55,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:55,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:55,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:55,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +14: [2023-05-13 09:38:55,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +14: [2023-05-13 09:38:55,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +14: [2023-05-13 09:38:55,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:55,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:55,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +24: [2023-05-13 09:38:55,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +24: [2023-05-13 09:38:55,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +24: [2023-05-13 09:38:55,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +24: [2023-05-13 09:38:55,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +11: [2023-05-13 09:38:55,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:55,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:55,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:55,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:55,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:55,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:55,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +15: [2023-05-13 09:38:55,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:55,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:55,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:55,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:55,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 0: [2023-05-13 09:38:55,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:55,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:55,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:55,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:55,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +15: [2023-05-13 09:38:55,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:55,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:55,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:55,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +25: [2023-05-13 09:38:55,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:55,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:55,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:55,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:55,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:55,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:55,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:55,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:55,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:55,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +25: [2023-05-13 09:38:55,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:55,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:55,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:55,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:55,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:55,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:55,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:55,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:55,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:55,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:55,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:55,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +14: [2023-05-13 09:38:55,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +13: [2023-05-13 09:38:55,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:55,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +17: [2023-05-13 09:38:55,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:55,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:55,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:55,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:55,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:55,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +17: [2023-05-13 09:38:55,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +13: [2023-05-13 09:38:55,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:55,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:55,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:55,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:55,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:55,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:55,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +14: [2023-05-13 09:38:55,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +25: [2023-05-13 09:38:55,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:55,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:55,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +25: [2023-05-13 09:38:55,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:55,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:55,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:55,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:55,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:55,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:55,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +26: [2023-05-13 09:38:55,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:55,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:55,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:55,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +26: [2023-05-13 09:38:55,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 0: [2023-05-13 09:38:55,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:55,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:55,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:55,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +21: [2023-05-13 09:38:55,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +21: [2023-05-13 09:38:55,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +17: [2023-05-13 09:38:55,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:55,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 0: [2023-05-13 09:38:55,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:55,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 7: [2023-05-13 09:38:55,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 7: [2023-05-13 09:38:55,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 7: [2023-05-13 09:38:55,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +24: [2023-05-13 09:38:55,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:55,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +26: [2023-05-13 09:38:55,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:55,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +26: [2023-05-13 09:38:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +26: [2023-05-13 09:38:55,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +26: [2023-05-13 09:38:55,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:55,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:55,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:55,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:55,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:55,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:55,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:55,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:55,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:55,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +29: [2023-05-13 09:38:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:55,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:55,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:55,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:55,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:55,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:55,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:55,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:55,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:55,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:55,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:55,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:55,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:55,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:55,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:55,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:55,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 3: [2023-05-13 09:38:55,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 9: [2023-05-13 09:38:55,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 9: [2023-05-13 09:38:55,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 9: [2023-05-13 09:38:55,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 3: [2023-05-13 09:38:55,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:55,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:55,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 3: [2023-05-13 09:38:55,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:55,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:55,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:55,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:55,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:55,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +29: [2023-05-13 09:38:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:55,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:55,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:55,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:55,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:55,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:55,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:55,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:55,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:55,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:55,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:55,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:55,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:55,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +20: [2023-05-13 09:38:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:55,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:55,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:55,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:55,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:55,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:55,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:55,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +27: [2023-05-13 09:38:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +27: [2023-05-13 09:38:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:55,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:55,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +30: [2023-05-13 09:38:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:55,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:55,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:55,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:55,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:55,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 9: [2023-05-13 09:38:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +12: [2023-05-13 09:38:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:55,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:55,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:55,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:55,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:55,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:55,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:55,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +16: [2023-05-13 09:38:55,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +20: [2023-05-13 09:38:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +16: [2023-05-13 09:38:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +22: [2023-05-13 09:38:55,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:55,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:55,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +22: [2023-05-13 09:38:55,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:55,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:55,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:55,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:55,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +16: [2023-05-13 09:38:55,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:55,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:55,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +27: [2023-05-13 09:38:55,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:55,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +16: [2023-05-13 09:38:55,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +21: [2023-05-13 09:38:55,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:55,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:55,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 7: [2023-05-13 09:38:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:55,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 4: [2023-05-13 09:38:55,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +16: [2023-05-13 09:38:55,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +19: [2023-05-13 09:38:55,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:55,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:55,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:55,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:55,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 4: [2023-05-13 09:38:55,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:55,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:55,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:55,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:55,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +19: [2023-05-13 09:38:55,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:55,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:55,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:55,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:55,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:55,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:55,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:55,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:55,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:55,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:55,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:55,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:55,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:55,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:55,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:55,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:55,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:55,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:55,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:55,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:55,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +13: [2023-05-13 09:38:55,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +13: [2023-05-13 09:38:55,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +13: [2023-05-13 09:38:55,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +12: [2023-05-13 09:38:55,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:55,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:55,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:55,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:55,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:55,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:55,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:55,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:55,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:55,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:55,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:55,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:55,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:55,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:55,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:55,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +13: [2023-05-13 09:38:55,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:55,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:55,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +20: [2023-05-13 09:38:55,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +20: [2023-05-13 09:38:55,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:55,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:55,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:55,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:55,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:55,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:55,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:55,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:55,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:55,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:55,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +13: [2023-05-13 09:38:55,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:55,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:55,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:55,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:55,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:55,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +19: [2023-05-13 09:38:55,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:55,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:55,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:55,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:55,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +30: [2023-05-13 09:38:55,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +20: [2023-05-13 09:38:55,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:55,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. + 8: [2023-05-13 09:38:55,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +16: [2023-05-13 09:38:55,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:55,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:55,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:55,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:55,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +28: [2023-05-13 09:38:55,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +20: [2023-05-13 09:38:55,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +20: [2023-05-13 09:38:55,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +16: [2023-05-13 09:38:55,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:55,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +19: [2023-05-13 09:38:55,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:55,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_24-model_01-model_states.pt. +20: [2023-05-13 09:38:55,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +30: [2023-05-13 09:38:55,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:55,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:55,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:55,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:55,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:55,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:55,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:55,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +28: [2023-05-13 09:38:55,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:55,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:55,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:55,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:55,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:55,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:55,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:55,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:55,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:55,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:55,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:55,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:55,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:55,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:55,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:55,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:55,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:55,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +13: [2023-05-13 09:38:55,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +13: [2023-05-13 09:38:55,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 8: [2023-05-13 09:38:55,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +15: [2023-05-13 09:38:55,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +15: [2023-05-13 09:38:55,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +15: [2023-05-13 09:38:55,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +15: [2023-05-13 09:38:55,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 1: [2023-05-13 09:38:55,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:55,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:55,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:55,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:55,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 8: [2023-05-13 09:38:55,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +14: [2023-05-13 09:38:55,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:55,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +14: [2023-05-13 09:38:55,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:55,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 8: [2023-05-13 09:38:55,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:55,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:55,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:55,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:55,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +28: [2023-05-13 09:38:55,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +15: [2023-05-13 09:38:55,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:55,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:55,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:55,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 0: [2023-05-13 09:38:55,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +15: [2023-05-13 09:38:55,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +15: [2023-05-13 09:38:55,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:55,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +24: [2023-05-13 09:38:55,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +24: [2023-05-13 09:38:55,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +24: [2023-05-13 09:38:55,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +24: [2023-05-13 09:38:55,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +17: [2023-05-13 09:38:55,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:55,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:55,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:55,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:55,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:55,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:55,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:55,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:55,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:55,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:55,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:55,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:55,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:55,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:55,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +25: [2023-05-13 09:38:55,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:55,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:55,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:55,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:55,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:55,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:55,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:55,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:55,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:55,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:55,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:55,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:55,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:55,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:55,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:55,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:55,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:55,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +14: [2023-05-13 09:38:55,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +30: [2023-05-13 09:38:55,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:55,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:55,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:55,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:55,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:55,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:55,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:55,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:55,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +15: [2023-05-13 09:38:55,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +15: [2023-05-13 09:38:55,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +14: [2023-05-13 09:38:55,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:55,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:55,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:55,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:55,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:55,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:55,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +27: [2023-05-13 09:38:55,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:55,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:55,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:55,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:55,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:55,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:55,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:55,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:55,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:55,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:55,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:55,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:55,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:55,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +27: [2023-05-13 09:38:55,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:55,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:55,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:55,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:55,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:55,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:55,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:55,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:55,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:55,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:55,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 0: [2023-05-13 09:38:55,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:55,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:55,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:55,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:55,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:55,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:55,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 0: [2023-05-13 09:38:55,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:55,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:55,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:55,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:55,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:55,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:55,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:55,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:55,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:55,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +24: [2023-05-13 09:38:55,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +24: [2023-05-13 09:38:55,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:55,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:55,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:55,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:55,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:55,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:55,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:55,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:55,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:55,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +30: [2023-05-13 09:38:55,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:55,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:55,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:55,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:55,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:55,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:55,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:55,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:55,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:55,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:55,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:55,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:55,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:55,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:55,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:55,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:55,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:55,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:55,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:55,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:55,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:55,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:56,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:56,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:56,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:56,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:56,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:56,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:56,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:56,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:56,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +27: [2023-05-13 09:38:56,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:56,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:56,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:56,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:56,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 1: [2023-05-13 09:38:56,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +16: [2023-05-13 09:38:56,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:56,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 1: [2023-05-13 09:38:56,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:56,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +12: [2023-05-13 09:38:56,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +16: [2023-05-13 09:38:56,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +16: [2023-05-13 09:38:56,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 6: [2023-05-13 09:38:56,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +13: [2023-05-13 09:38:56,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +13: [2023-05-13 09:38:56,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +13: [2023-05-13 09:38:56,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +13: [2023-05-13 09:38:56,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +20: [2023-05-13 09:38:56,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +20: [2023-05-13 09:38:56,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +20: [2023-05-13 09:38:56,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:56,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:56,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:56,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:56,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:56,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:56,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:56,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:56,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:56,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:56,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:56,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:56,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:56,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:56,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:56,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:56,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:56,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:56,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:56,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:56,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:56,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:56,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:56,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:56,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:56,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +29: [2023-05-13 09:38:56,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:56,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +29: [2023-05-13 09:38:56,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:56,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:56,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:56,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:56,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:56,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:56,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:56,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +29: [2023-05-13 09:38:56,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:56,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:56,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:56,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:56,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:56,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:56,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:56,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +26: [2023-05-13 09:38:56,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:56,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:56,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:56,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +12: [2023-05-13 09:38:56,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:56,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:56,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:56,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:56,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:56,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:56,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +16: [2023-05-13 09:38:56,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +12: [2023-05-13 09:38:56,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +20: [2023-05-13 09:38:56,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +16: [2023-05-13 09:38:56,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:56,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +20: [2023-05-13 09:38:56,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 8: [2023-05-13 09:38:56,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:56,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:56,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:56,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:56,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:56,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:56,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:56,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:56,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:56,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:56,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:56,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:56,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:56,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:56,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:56,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +20: [2023-05-13 09:38:56,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:56,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:56,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:56,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:56,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:56,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +14: [2023-05-13 09:38:56,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:56,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 8: [2023-05-13 09:38:56,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:56,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:56,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:56,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:56,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:56,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:56,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:56,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +13: [2023-05-13 09:38:56,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +15: [2023-05-13 09:38:56,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:56,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +16: [2023-05-13 09:38:56,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:56,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:56,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:56,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:56,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:56,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:56,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:56,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:56,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:56,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:56,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:56,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:56,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:56,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:56,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:56,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:56,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:56,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:56,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:56,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:56,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:56,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:56,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +31: [2023-05-13 09:38:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:56,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:56,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +31: [2023-05-13 09:38:56,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:56,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:56,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:56,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:56,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +14: [2023-05-13 09:38:56,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:56,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:56,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:56,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:56,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:56,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:56,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:56,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:56,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:56,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:56,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:56,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +29: [2023-05-13 09:38:56,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:56,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 8: [2023-05-13 09:38:56,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:56,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:56,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:56,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:56,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:56,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:56,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:56,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +29: [2023-05-13 09:38:56,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +10: [2023-05-13 09:38:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +29: [2023-05-13 09:38:56,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +31: [2023-05-13 09:38:56,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:56,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +31: [2023-05-13 09:38:56,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +28: [2023-05-13 09:38:56,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:56,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:56,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 8: [2023-05-13 09:38:56,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:56,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +24: [2023-05-13 09:38:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:56,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +10: [2023-05-13 09:38:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +10: [2023-05-13 09:38:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +16: [2023-05-13 09:38:56,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +25: [2023-05-13 09:38:56,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:56,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:56,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:56,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:56,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +26: [2023-05-13 09:38:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +26: [2023-05-13 09:38:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:56,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +25: [2023-05-13 09:38:56,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:56,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:56,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:56,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:56,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:56,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:56,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:56,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:56,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:56,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +21: [2023-05-13 09:38:56,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +16: [2023-05-13 09:38:56,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:56,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:56,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:56,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:56,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +24: [2023-05-13 09:38:56,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:56,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +26: [2023-05-13 09:38:56,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:56,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:56,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:56,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +19: [2023-05-13 09:38:56,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +19: [2023-05-13 09:38:56,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +19: [2023-05-13 09:38:56,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +29: [2023-05-13 09:38:56,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +30: [2023-05-13 09:38:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +30: [2023-05-13 09:38:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +30: [2023-05-13 09:38:56,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +19: [2023-05-13 09:38:56,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:56,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:56,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +29: [2023-05-13 09:38:56,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:56,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 8: [2023-05-13 09:38:56,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:56,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +21: [2023-05-13 09:38:56,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:56,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:56,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +21: [2023-05-13 09:38:56,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:56,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:56,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:56,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:56,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:56,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +26: [2023-05-13 09:38:56,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +28: [2023-05-13 09:38:56,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:56,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:56,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +28: [2023-05-13 09:38:56,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +28: [2023-05-13 09:38:56,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:56,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:56,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:56,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:56,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:56,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:56,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:56,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +22: [2023-05-13 09:38:56,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:56,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +22: [2023-05-13 09:38:56,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +29: [2023-05-13 09:38:56,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:56,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +30: [2023-05-13 09:38:56,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:56,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:56,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +30: [2023-05-13 09:38:56,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:56,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:56,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +20: [2023-05-13 09:38:56,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:56,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:56,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:56,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:56,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:56,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:56,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:56,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:56,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +10: [2023-05-13 09:38:56,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:56,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:56,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:56,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:56,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:56,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:56,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:56,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +28: [2023-05-13 09:38:56,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:56,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:56,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:56,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:56,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:56,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:56,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:56,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:56,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:56,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:56,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:56,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +21: [2023-05-13 09:38:56,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:56,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +17: [2023-05-13 09:38:56,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +17: [2023-05-13 09:38:56,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:56,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +11: [2023-05-13 09:38:56,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:56,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:56,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:56,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:56,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:56,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +21: [2023-05-13 09:38:56,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:56,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:56,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:56,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:56,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:56,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +11: [2023-05-13 09:38:56,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:56,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +11: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +19: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +11: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +11: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +18: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +18: [2023-05-13 09:38:56,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:56,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +20: [2023-05-13 09:38:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +19: [2023-05-13 09:38:56,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +19: [2023-05-13 09:38:56,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +22: [2023-05-13 09:38:56,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:56,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +22: [2023-05-13 09:38:56,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +22: [2023-05-13 09:38:56,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:56,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:56,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:56,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:56,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:56,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:56,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:56,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:56,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:56,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:56,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:56,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:56,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:56,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:56,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:56,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +15: [2023-05-13 09:38:56,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:56,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 7: [2023-05-13 09:38:56,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:56,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 7: [2023-05-13 09:38:56,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +14: [2023-05-13 09:38:56,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:56,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:56,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:56,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:56,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +15: [2023-05-13 09:38:56,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:56,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:56,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:56,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +15: [2023-05-13 09:38:56,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +20: [2023-05-13 09:38:56,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:56,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:56,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:56,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:56,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +28: [2023-05-13 09:38:56,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:56,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:56,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +22: [2023-05-13 09:38:56,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +28: [2023-05-13 09:38:56,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:56,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:56,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 7: [2023-05-13 09:38:56,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:56,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:56,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:56,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:56,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:56,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:56,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:56,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:56,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:56,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:56,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:56,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:56,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:56,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:56,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:56,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:56,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +14: [2023-05-13 09:38:56,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:56,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:56,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:56,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:56,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:56,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:56,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:56,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:56,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:56,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +18: [2023-05-13 09:38:56,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:56,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:56,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +31: [2023-05-13 09:38:56,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +31: [2023-05-13 09:38:56,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:56,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:56,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:56,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:56,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:56,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:56,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:56,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +25: [2023-05-13 09:38:56,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:56,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +25: [2023-05-13 09:38:56,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +11: [2023-05-13 09:38:56,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +11: [2023-05-13 09:38:56,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:56,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:56,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:56,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:56,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:56,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:56,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:56,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:56,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:56,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:56,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:56,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:56,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:56,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:56,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:56,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:56,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:56,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:56,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:56,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:56,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +12: [2023-05-13 09:38:56,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +12: [2023-05-13 09:38:56,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +12: [2023-05-13 09:38:56,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:56,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:56,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:56,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:56,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:56,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:56,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:56,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:56,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:56,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:56,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +22: [2023-05-13 09:38:56,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +22: [2023-05-13 09:38:56,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:56,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:56,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +17: [2023-05-13 09:38:56,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +14: [2023-05-13 09:38:56,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +22: [2023-05-13 09:38:56,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:56,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:56,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:56,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 5: [2023-05-13 09:38:56,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 5: [2023-05-13 09:38:56,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... + 5: [2023-05-13 09:38:56,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +31: [2023-05-13 09:38:56,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:56,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:56,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:56,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:56,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:56,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +14: [2023-05-13 09:38:56,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:56,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:56,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:56,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:56,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:56,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:56,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 0: [2023-05-13 09:38:56,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:56,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +12: [2023-05-13 09:38:56,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:56,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:56,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +17: [2023-05-13 09:38:56,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 0: [2023-05-13 09:38:56,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:56,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:56,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:56,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:56,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 0: [2023-05-13 09:38:56,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:56,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:56,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:56,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:56,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:56,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:56,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:56,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:56,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:56,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:56,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +31: [2023-05-13 09:38:56,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:56,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:56,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:56,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:56,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:56,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:56,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:56,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:56,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:56,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:56,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:56,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:56,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +22: [2023-05-13 09:38:56,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:56,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:56,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +27: [2023-05-13 09:38:56,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:56,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:56,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +25: [2023-05-13 09:38:56,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +17: [2023-05-13 09:38:56,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:56,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +27: [2023-05-13 09:38:56,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +22: [2023-05-13 09:38:56,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:56,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:56,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:56,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:56,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:56,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:56,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:56,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +12: [2023-05-13 09:38:56,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:56,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +10: [2023-05-13 09:38:56,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +22: [2023-05-13 09:38:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +10: [2023-05-13 09:38:56,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +12: [2023-05-13 09:38:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +10: [2023-05-13 09:38:56,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +12: [2023-05-13 09:38:56,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +21: [2023-05-13 09:38:56,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:56,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:56,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:56,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:56,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +30: [2023-05-13 09:38:56,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:56,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:56,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:56,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:56,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:56,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:56,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:56,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:56,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:56,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:56,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:56,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +24: [2023-05-13 09:38:56,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:56,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:56,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:56,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:56,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:56,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:56,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:56,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +26: [2023-05-13 09:38:56,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:56,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:56,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:56,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +26: [2023-05-13 09:38:56,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +14: [2023-05-13 09:38:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:56,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:56,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +12: [2023-05-13 09:38:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +12: [2023-05-13 09:38:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +12: [2023-05-13 09:38:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt... +23: [2023-05-13 09:38:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +23: [2023-05-13 09:38:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt... +20: [2023-05-13 09:38:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +21: [2023-05-13 09:38:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:56,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +20: [2023-05-13 09:38:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +24: [2023-05-13 09:38:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:56,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +15: [2023-05-13 09:38:56,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:56,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:56,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:56,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:56,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:56,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:56,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +14: [2023-05-13 09:38:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:56,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +25: [2023-05-13 09:38:56,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:56,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:56,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:56,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +30: [2023-05-13 09:38:56,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +16: [2023-05-13 09:38:56,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +25: [2023-05-13 09:38:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +25: [2023-05-13 09:38:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +10: [2023-05-13 09:38:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +16: [2023-05-13 09:38:56,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +13: [2023-05-13 09:38:56,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:56,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:56,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 1: [2023-05-13 09:38:56,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:56,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +26: [2023-05-13 09:38:56,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 1: [2023-05-13 09:38:56,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:56,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:56,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:56,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +14: [2023-05-13 09:38:56,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +14: [2023-05-13 09:38:56,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:56,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:56,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:56,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. +23: [2023-05-13 09:38:56,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +15: [2023-05-13 09:38:56,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:56,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:56,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:56,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:56,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:56,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:56,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:56,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:56,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:56,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:56,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:56,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:56,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 8: [2023-05-13 09:38:56,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +13: [2023-05-13 09:38:56,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +13: [2023-05-13 09:38:56,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +13: [2023-05-13 09:38:56,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:56,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:56,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:56,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:56,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:56,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:56,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +26: [2023-05-13 09:38:56,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:56,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:56,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:56,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 8: [2023-05-13 09:38:56,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +23: [2023-05-13 09:38:56,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:56,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +29: [2023-05-13 09:38:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:56,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:56,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:56,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:56,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:56,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:56,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:56,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:56,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +25: [2023-05-13 09:38:56,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:56,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:56,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:56,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:56,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +18: [2023-05-13 09:38:56,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:56,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:56,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:56,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:56,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:56,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +13: [2023-05-13 09:38:56,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +13: [2023-05-13 09:38:56,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +16: [2023-05-13 09:38:56,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:56,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:56,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:56,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:56,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:56,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:56,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:56,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:56,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +29: [2023-05-13 09:38:56,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:56,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:56,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:56,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:56,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:56,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +11: [2023-05-13 09:38:56,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +16: [2023-05-13 09:38:56,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:56,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +26: [2023-05-13 09:38:56,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:56,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:56,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:56,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:56,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:56,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +11: [2023-05-13 09:38:56,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +25: [2023-05-13 09:38:56,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +28: [2023-05-13 09:38:56,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +28: [2023-05-13 09:38:56,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +26: [2023-05-13 09:38:56,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +24: [2023-05-13 09:38:56,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:56,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:56,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:56,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:56,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +28: [2023-05-13 09:38:56,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:56,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:56,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:56,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +31: [2023-05-13 09:38:56,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +24: [2023-05-13 09:38:56,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +26: [2023-05-13 09:38:56,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:56,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +21: [2023-05-13 09:38:56,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +21: [2023-05-13 09:38:56,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:56,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:56,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:56,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +29: [2023-05-13 09:38:56,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +13: [2023-05-13 09:38:56,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:56,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:56,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +16: [2023-05-13 09:38:56,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +19: [2023-05-13 09:38:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:56,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:56,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:56,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:56,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:56,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:56,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +19: [2023-05-13 09:38:56,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:56,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:56,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:56,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:56,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:56,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:56,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:56,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:56,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:56,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +29: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +21: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 5: [2023-05-13 09:38:56,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 5: [2023-05-13 09:38:56,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +21: [2023-05-13 09:38:56,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 5: [2023-05-13 09:38:56,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:56,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:56,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:56,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +17: [2023-05-13 09:38:56,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:56,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:56,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:56,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +13: [2023-05-13 09:38:56,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:56,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:56,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:56,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:56,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:56,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:56,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +15: [2023-05-13 09:38:56,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +15: [2023-05-13 09:38:56,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +17: [2023-05-13 09:38:56,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +21: [2023-05-13 09:38:56,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:56,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:56,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:56,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:56,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:56,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:56,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:56,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:56,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:56,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +17: [2023-05-13 09:38:56,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +17: [2023-05-13 09:38:56,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:56,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:56,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:56,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +11: [2023-05-13 09:38:56,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 5: [2023-05-13 09:38:56,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:56,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +15: [2023-05-13 09:38:56,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +17: [2023-05-13 09:38:56,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +21: [2023-05-13 09:38:56,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:56,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +24: [2023-05-13 09:38:56,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:56,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:56,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:56,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:56,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 7: [2023-05-13 09:38:56,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 7: [2023-05-13 09:38:56,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:56,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +17: [2023-05-13 09:38:56,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:56,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:56,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:56,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:56,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:56,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:56,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:56,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:56,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:56,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:56,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:56,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:56,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:56,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:56,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +18: [2023-05-13 09:38:56,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:56,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:56,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +29: [2023-05-13 09:38:56,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:56,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 5: [2023-05-13 09:38:56,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:56,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +17: [2023-05-13 09:38:56,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:56,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:56,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:56,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:56,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:56,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +18: [2023-05-13 09:38:56,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +17: [2023-05-13 09:38:56,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:56,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:56,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:56,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:56,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +17: [2023-05-13 09:38:56,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:56,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:56,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:56,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:56,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:56,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:56,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +19: [2023-05-13 09:38:56,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:56,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:56,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:56,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:56,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:56,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:56,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:56,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:56,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:56,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:56,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:56,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:56,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:56,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:56,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +28: [2023-05-13 09:38:56,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:56,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +31: [2023-05-13 09:38:56,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:56,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:56,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +28: [2023-05-13 09:38:56,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +27: [2023-05-13 09:38:56,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 5: [2023-05-13 09:38:56,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:56,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:56,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:56,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +31: [2023-05-13 09:38:56,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:56,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:56,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:56,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +14: [2023-05-13 09:38:56,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +18: [2023-05-13 09:38:56,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +17: [2023-05-13 09:38:56,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:56,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:56,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 5: [2023-05-13 09:38:56,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 5: [2023-05-13 09:38:56,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +15: [2023-05-13 09:38:56,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:56,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:56,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +15: [2023-05-13 09:38:56,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:56,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:56,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:56,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:56,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:56,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:56,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:56,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. +23: [2023-05-13 09:38:56,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_25-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:56,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +25: [2023-05-13 09:38:56,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:56,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:56,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:56,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +27: [2023-05-13 09:38:56,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:56,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:56,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:56,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:56,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:56,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +22: [2023-05-13 09:38:56,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:56,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:56,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:56,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:56,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:56,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:56,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:56,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +11: [2023-05-13 09:38:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +14: [2023-05-13 09:38:56,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +30: [2023-05-13 09:38:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +30: [2023-05-13 09:38:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:56,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +19: [2023-05-13 09:38:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:56,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:56,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +11: [2023-05-13 09:38:56,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +19: [2023-05-13 09:38:56,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:56,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:56,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:56,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:56,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:56,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:56,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:56,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:56,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:56,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +17: [2023-05-13 09:38:56,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:56,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:56,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +27: [2023-05-13 09:38:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +25: [2023-05-13 09:38:56,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:56,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:56,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:56,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:56,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:56,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:56,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:56,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:56,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:56,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:56,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:56,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:56,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:56,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:56,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +17: [2023-05-13 09:38:56,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +17: [2023-05-13 09:38:56,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:56,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:56,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:56,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:56,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:56,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 0: [2023-05-13 09:38:56,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +19: [2023-05-13 09:38:56,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:56,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:56,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:56,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:56,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:56,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:56,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:56,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:56,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:56,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:56,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:56,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:56,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:56,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +20: [2023-05-13 09:38:56,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:56,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:56,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:56,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +21: [2023-05-13 09:38:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:56,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:56,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:56,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:56,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:56,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:56,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:56,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:56,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:56,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:56,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:56,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:56,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:56,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:56,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:56,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:56,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:56,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:56,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:56,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:56,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +25: [2023-05-13 09:38:56,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +25: [2023-05-13 09:38:56,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:56,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:56,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:56,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:56,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:56,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:56,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:56,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:56,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:56,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:56,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:56,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:56,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:56,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:56,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:56,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +30: [2023-05-13 09:38:56,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +30: [2023-05-13 09:38:56,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:56,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +22: [2023-05-13 09:38:56,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:56,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +12: [2023-05-13 09:38:56,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +12: [2023-05-13 09:38:56,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +12: [2023-05-13 09:38:56,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +23: [2023-05-13 09:38:56,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:56,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:56,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +12: [2023-05-13 09:38:56,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +23: [2023-05-13 09:38:56,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:56,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:56,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:56,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:56,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:56,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +20: [2023-05-13 09:38:56,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:56,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:56,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:56,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +18: [2023-05-13 09:38:56,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +22: [2023-05-13 09:38:56,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 6: [2023-05-13 09:38:56,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +23: [2023-05-13 09:38:56,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +23: [2023-05-13 09:38:56,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +18: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +23: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... +23: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +23: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +18: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +10: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +21: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +23: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt... +10: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:56,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:56,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +10: [2023-05-13 09:38:56,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:56,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:56,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 1: [2023-05-13 09:38:56,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +21: [2023-05-13 09:38:56,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 1: [2023-05-13 09:38:56,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +20: [2023-05-13 09:38:56,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +12: [2023-05-13 09:38:56,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:56,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +20: [2023-05-13 09:38:56,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:56,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:56,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:56,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:56,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:56,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:56,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:56,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:56,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 0: [2023-05-13 09:38:56,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:56,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:56,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +20: [2023-05-13 09:38:56,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:56,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:56,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:56,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:56,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:56,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:56,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +20: [2023-05-13 09:38:56,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:56,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:56,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:56,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:56,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:56,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +10: [2023-05-13 09:38:56,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:56,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:56,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:56,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:56,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:56,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:56,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. +23: [2023-05-13 09:38:56,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:56,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +27: [2023-05-13 09:38:56,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:56,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:56,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:56,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +12: [2023-05-13 09:38:56,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:56,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:56,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:56,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:56,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:56,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +10: [2023-05-13 09:38:56,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:56,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +20: [2023-05-13 09:38:56,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:56,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:56,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:56,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:56,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 6: [2023-05-13 09:38:56,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 8: [2023-05-13 09:38:56,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 1: [2023-05-13 09:38:56,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:56,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:56,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:56,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +12: [2023-05-13 09:38:56,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:56,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:56,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:56,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:56,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +12: [2023-05-13 09:38:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:56,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:56,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 5: [2023-05-13 09:38:56,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 5: [2023-05-13 09:38:56,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 5: [2023-05-13 09:38:56,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +27: [2023-05-13 09:38:56,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +27: [2023-05-13 09:38:56,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:56,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +26: [2023-05-13 09:38:56,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:56,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +11: [2023-05-13 09:38:56,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +11: [2023-05-13 09:38:56,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:56,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +15: [2023-05-13 09:38:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:56,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:56,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:56,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +27: [2023-05-13 09:38:56,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +27: [2023-05-13 09:38:56,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:56,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:56,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:56,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:56,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:56,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +25: [2023-05-13 09:38:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:56,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +26: [2023-05-13 09:38:56,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 5: [2023-05-13 09:38:56,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:56,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +25: [2023-05-13 09:38:56,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:56,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:56,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:56,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:56,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:56,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:56,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:56,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:56,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:56,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:56,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:56,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:56,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:56,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +13: [2023-05-13 09:38:56,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:56,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:56,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:56,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:56,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:56,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:56,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:56,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:56,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:56,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:56,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:56,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:56,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:56,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:56,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:56,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:56,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:56,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:56,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:56,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +16: [2023-05-13 09:38:56,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:56,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:56,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:56,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:56,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:56,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:56,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:56,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:56,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:56,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:56,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:56,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:56,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:56,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:56,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:56,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +18: [2023-05-13 09:38:56,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +18: [2023-05-13 09:38:56,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +12: [2023-05-13 09:38:56,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:56,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:56,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:56,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +12: [2023-05-13 09:38:56,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:56,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:56,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:56,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +30: [2023-05-13 09:38:56,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:56,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:56,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:56,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:56,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:56,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:56,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:56,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:56,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:56,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:56,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:56,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:56,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:56,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:56,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +17: [2023-05-13 09:38:56,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:56,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:56,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:56,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:56,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:56,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:56,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:56,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:56,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:56,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:56,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:56,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +15: [2023-05-13 09:38:56,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:56,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:56,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:56,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:56,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +16: [2023-05-13 09:38:56,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:56,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:56,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:56,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +30: [2023-05-13 09:38:56,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:56,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +15: [2023-05-13 09:38:56,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:56,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:56,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +13: [2023-05-13 09:38:56,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:56,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:56,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +30: [2023-05-13 09:38:56,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:56,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +11: [2023-05-13 09:38:56,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +11: [2023-05-13 09:38:56,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +16: [2023-05-13 09:38:56,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 5: [2023-05-13 09:38:56,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 5: [2023-05-13 09:38:56,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:56,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:56,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:56,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:56,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:56,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +29: [2023-05-13 09:38:56,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:56,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:56,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:56,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +16: [2023-05-13 09:38:56,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:56,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +13: [2023-05-13 09:38:56,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:56,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:56,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:56,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:56,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:56,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:56,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +16: [2023-05-13 09:38:56,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +29: [2023-05-13 09:38:56,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:56,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:56,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:56,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:56,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +18: [2023-05-13 09:38:56,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +25: [2023-05-13 09:38:56,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:56,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:56,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:56,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:56,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:56,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:56,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:56,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:56,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:56,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:56,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +30: [2023-05-13 09:38:56,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:56,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +31: [2023-05-13 09:38:56,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:56,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:56,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +14: [2023-05-13 09:38:56,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:56,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:56,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:56,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:56,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:56,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:56,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:56,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +24: [2023-05-13 09:38:56,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:56,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:56,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +13: [2023-05-13 09:38:56,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +13: [2023-05-13 09:38:56,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +17: [2023-05-13 09:38:56,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:56,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:56,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:56,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:56,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:56,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:56,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +28: [2023-05-13 09:38:56,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:56,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:56,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:56,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:56,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:56,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:56,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:56,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:56,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:56,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:56,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:56,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:56,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:56,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:56,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:56,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:56,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +11: [2023-05-13 09:38:56,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +29: [2023-05-13 09:38:56,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +29: [2023-05-13 09:38:56,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:56,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:56,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +31: [2023-05-13 09:38:56,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:56,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:56,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:56,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:56,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:56,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +27: [2023-05-13 09:38:56,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:56,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:56,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:56,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:56,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:56,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:56,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +11: [2023-05-13 09:38:56,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +17: [2023-05-13 09:38:56,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:56,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:56,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:56,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:56,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:56,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:56,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +13: [2023-05-13 09:38:56,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:56,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:56,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +29: [2023-05-13 09:38:56,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:56,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +28: [2023-05-13 09:38:56,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:56,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:56,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +14: [2023-05-13 09:38:56,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:56,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:56,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:56,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:56,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:56,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +17: [2023-05-13 09:38:56,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:56,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:56,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:56,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:56,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:56,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +23: [2023-05-13 09:38:56,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +29: [2023-05-13 09:38:56,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:56,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +31: [2023-05-13 09:38:56,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:56,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:56,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:56,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +23: [2023-05-13 09:38:56,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:56,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +23: [2023-05-13 09:38:56,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:56,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:56,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:56,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:56,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:56,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:56,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 7: [2023-05-13 09:38:56,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:56,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:56,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:56,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:56,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +23: [2023-05-13 09:38:56,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +23: [2023-05-13 09:38:56,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. +29: [2023-05-13 09:38:56,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +29: [2023-05-13 09:38:56,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:56,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_26-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:56,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:56,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:56,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +13: [2023-05-13 09:38:56,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +29: [2023-05-13 09:38:56,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +13: [2023-05-13 09:38:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +10: [2023-05-13 09:38:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +10: [2023-05-13 09:38:56,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +17: [2023-05-13 09:38:56,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:56,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +17: [2023-05-13 09:38:56,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +17: [2023-05-13 09:38:56,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +21: [2023-05-13 09:38:56,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +21: [2023-05-13 09:38:56,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:56,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:56,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +17: [2023-05-13 09:38:56,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:56,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:56,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:56,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +21: [2023-05-13 09:38:56,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:56,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +21: [2023-05-13 09:38:56,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:56,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +13: [2023-05-13 09:38:56,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +19: [2023-05-13 09:38:56,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +26: [2023-05-13 09:38:56,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +26: [2023-05-13 09:38:56,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +26: [2023-05-13 09:38:56,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +24: [2023-05-13 09:38:56,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:56,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:56,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +15: [2023-05-13 09:38:56,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +15: [2023-05-13 09:38:56,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:56,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:56,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:56,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:56,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +13: [2023-05-13 09:38:56,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +24: [2023-05-13 09:38:56,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:56,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +24: [2023-05-13 09:38:56,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:56,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:56,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:56,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:56,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +24: [2023-05-13 09:38:56,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:56,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:56,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +24: [2023-05-13 09:38:56,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:56,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:56,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:56,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +27: [2023-05-13 09:38:56,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:56,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:56,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:56,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 0: [2023-05-13 09:38:56,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +28: [2023-05-13 09:38:56,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 7: [2023-05-13 09:38:56,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +28: [2023-05-13 09:38:56,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 1: [2023-05-13 09:38:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 1: [2023-05-13 09:38:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +23: [2023-05-13 09:38:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +23: [2023-05-13 09:38:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... +28: [2023-05-13 09:38:56,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +28: [2023-05-13 09:38:56,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +14: [2023-05-13 09:38:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:56,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +26: [2023-05-13 09:38:56,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:56,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:56,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +21: [2023-05-13 09:38:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +26: [2023-05-13 09:38:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +19: [2023-05-13 09:38:56,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:56,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:56,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +17: [2023-05-13 09:38:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +15: [2023-05-13 09:38:56,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:56,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:56,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:56,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:56,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:56,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:56,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:56,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:56,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:56,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:56,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:56,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:56,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:56,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 7: [2023-05-13 09:38:56,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:56,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 9: [2023-05-13 09:38:56,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +25: [2023-05-13 09:38:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 7: [2023-05-13 09:38:56,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:56,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:56,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:56,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +24: [2023-05-13 09:38:56,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:56,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:56,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +22: [2023-05-13 09:38:56,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 4: [2023-05-13 09:38:56,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:56,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:56,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:56,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +17: [2023-05-13 09:38:56,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:56,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +22: [2023-05-13 09:38:56,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +23: [2023-05-13 09:38:56,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:56,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +24: [2023-05-13 09:38:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +14: [2023-05-13 09:38:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +12: [2023-05-13 09:38:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +12: [2023-05-13 09:38:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +12: [2023-05-13 09:38:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +12: [2023-05-13 09:38:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +15: [2023-05-13 09:38:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:56,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 8: [2023-05-13 09:38:56,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +15: [2023-05-13 09:38:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:56,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:56,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:56,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:56,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +26: [2023-05-13 09:38:56,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:56,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +15: [2023-05-13 09:38:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +15: [2023-05-13 09:38:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +21: [2023-05-13 09:38:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +25: [2023-05-13 09:38:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_00-model_states.pt. +24: [2023-05-13 09:38:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +10: [2023-05-13 09:38:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +18: [2023-05-13 09:38:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +19: [2023-05-13 09:38:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt... +10: [2023-05-13 09:38:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +31: [2023-05-13 09:38:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +18: [2023-05-13 09:38:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:56,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +10: [2023-05-13 09:38:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +17: [2023-05-13 09:38:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +17: [2023-05-13 09:38:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 0: [2023-05-13 09:38:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 0: [2023-05-13 09:38:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +21: [2023-05-13 09:38:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +12: [2023-05-13 09:38:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +10: [2023-05-13 09:38:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +19: [2023-05-13 09:38:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +10: [2023-05-13 09:38:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +22: [2023-05-13 09:38:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:56,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:56,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 0: [2023-05-13 09:38:56,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:56,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +27: [2023-05-13 09:38:56,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +27: [2023-05-13 09:38:56,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +27: [2023-05-13 09:38:56,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +19: [2023-05-13 09:38:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:56,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:56,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 0: [2023-05-13 09:38:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:56,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +12: [2023-05-13 09:38:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:56,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:56,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:56,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:56,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:56,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:56,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:56,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:56,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:56,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:56,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:56,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:56,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:56,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:56,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:56,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:56,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:56,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +27: [2023-05-13 09:38:56,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:56,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:56,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:56,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:56,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:56,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:56,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 4: [2023-05-13 09:38:56,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:56,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:56,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:56,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:56,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:56,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:56,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +25: [2023-05-13 09:38:56,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:56,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:56,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:56,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:56,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:56,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:56,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:56,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:56,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:56,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:56,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:56,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:56,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:56,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:56,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:56,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:56,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:56,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 6: [2023-05-13 09:38:56,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:56,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 6: [2023-05-13 09:38:56,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:56,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 9: [2023-05-13 09:38:56,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:56,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:56,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:56,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:56,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:56,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:56,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:56,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:56,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +12: [2023-05-13 09:38:56,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +12: [2023-05-13 09:38:56,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +27: [2023-05-13 09:38:56,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:56,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:56,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:56,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +27: [2023-05-13 09:38:56,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:56,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:56,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:56,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:56,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:56,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:56,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:56,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:56,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:56,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +24: [2023-05-13 09:38:56,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +24: [2023-05-13 09:38:56,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +24: [2023-05-13 09:38:56,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:56,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:56,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +27: [2023-05-13 09:38:56,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +27: [2023-05-13 09:38:56,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:56,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 1: [2023-05-13 09:38:56,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:56,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +22: [2023-05-13 09:38:56,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:56,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:56,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:56,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:56,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:56,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:56,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:56,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:56,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:56,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +24: [2023-05-13 09:38:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:56,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +24: [2023-05-13 09:38:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:56,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:56,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:56,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +11: [2023-05-13 09:38:56,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +11: [2023-05-13 09:38:56,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +11: [2023-05-13 09:38:56,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 2: [2023-05-13 09:38:56,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:57,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:57,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:57,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:57,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:57,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +22: [2023-05-13 09:38:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:57,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 9: [2023-05-13 09:38:57,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:57,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +22: [2023-05-13 09:38:57,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 9: [2023-05-13 09:38:57,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +22: [2023-05-13 09:38:57,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:57,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:57,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:57,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:57,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:57,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:57,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:57,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:57,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:57,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:57,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:57,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:57,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:57,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:57,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:57,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:57,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +18: [2023-05-13 09:38:57,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +11: [2023-05-13 09:38:57,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +11: [2023-05-13 09:38:57,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:57,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:57,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:57,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:57,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +11: [2023-05-13 09:38:57,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +21: [2023-05-13 09:38:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:57,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:57,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:57,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:57,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:57,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:57,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +21: [2023-05-13 09:38:57,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:57,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +16: [2023-05-13 09:38:57,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:57,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:57,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:57,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:57,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 3: [2023-05-13 09:38:57,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:57,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +20: [2023-05-13 09:38:57,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:57,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:57,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:57,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +23: [2023-05-13 09:38:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +20: [2023-05-13 09:38:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +16: [2023-05-13 09:38:57,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:57,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:57,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:57,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:57,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:57,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +16: [2023-05-13 09:38:57,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:57,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:57,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +24: [2023-05-13 09:38:57,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +24: [2023-05-13 09:38:57,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +20: [2023-05-13 09:38:57,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:57,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +18: [2023-05-13 09:38:57,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:57,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +24: [2023-05-13 09:38:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +20: [2023-05-13 09:38:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +20: [2023-05-13 09:38:57,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:57,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:57,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:57,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +28: [2023-05-13 09:38:57,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +20: [2023-05-13 09:38:57,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +21: [2023-05-13 09:38:57,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:57,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +28: [2023-05-13 09:38:57,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +20: [2023-05-13 09:38:57,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +28: [2023-05-13 09:38:57,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:57,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +18: [2023-05-13 09:38:57,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:57,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:57,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:57,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +11: [2023-05-13 09:38:57,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:57,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:57,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:57,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:57,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:57,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +11: [2023-05-13 09:38:57,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:57,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +23: [2023-05-13 09:38:57,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +13: [2023-05-13 09:38:57,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +13: [2023-05-13 09:38:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +13: [2023-05-13 09:38:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +23: [2023-05-13 09:38:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +20: [2023-05-13 09:38:57,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +23: [2023-05-13 09:38:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +13: [2023-05-13 09:38:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:57,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:57,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +11: [2023-05-13 09:38:57,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 9: [2023-05-13 09:38:57,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:57,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:57,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:57,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_27-model_01-model_states.pt. +11: [2023-05-13 09:38:57,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:57,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:57,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:57,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:57,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:57,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:57,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:57,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +21: [2023-05-13 09:38:57,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:57,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:57,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:57,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:57,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:57,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:57,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +11: [2023-05-13 09:38:57,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:57,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +31: [2023-05-13 09:38:57,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:57,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:57,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:57,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:57,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:57,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:57,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 8: [2023-05-13 09:38:57,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:57,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:57,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:57,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +16: [2023-05-13 09:38:57,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:57,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:57,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:57,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 6: [2023-05-13 09:38:57,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +23: [2023-05-13 09:38:57,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:57,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:57,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +10: [2023-05-13 09:38:57,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +12: [2023-05-13 09:38:57,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:57,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:57,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:57,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:57,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:57,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:57,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:57,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:57,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:57,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:57,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:57,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:57,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:57,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:57,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +12: [2023-05-13 09:38:57,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:57,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:57,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt... +14: [2023-05-13 09:38:57,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +14: [2023-05-13 09:38:57,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:57,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:57,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +26: [2023-05-13 09:38:57,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:57,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:57,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:57,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:57,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:57,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:57,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:57,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:57,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:57,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +28: [2023-05-13 09:38:57,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +10: [2023-05-13 09:38:57,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:57,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:57,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:57,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:57,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:57,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:57,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:57,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:57,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:57,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:57,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +16: [2023-05-13 09:38:57,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:57,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:57,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:57,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:57,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:57,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:57,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:57,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:57,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:57,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +31: [2023-05-13 09:38:57,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:57,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +16: [2023-05-13 09:38:57,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:57,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:57,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:57,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:57,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +12: [2023-05-13 09:38:57,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:57,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +30: [2023-05-13 09:38:57,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +14: [2023-05-13 09:38:57,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:57,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:57,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +31: [2023-05-13 09:38:57,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:57,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:57,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +12: [2023-05-13 09:38:57,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:57,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... +31: [2023-05-13 09:38:57,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +31: [2023-05-13 09:38:57,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:57,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:57,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:57,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:57,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +13: [2023-05-13 09:38:57,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +14: [2023-05-13 09:38:57,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:57,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +12: [2023-05-13 09:38:57,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +23: [2023-05-13 09:38:57,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_00-model_states.pt. +14: [2023-05-13 09:38:57,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:57,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +13: [2023-05-13 09:38:57,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +13: [2023-05-13 09:38:57,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:57,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:57,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:57,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:57,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:57,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:57,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:57,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +16: [2023-05-13 09:38:57,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:57,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:57,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +14: [2023-05-13 09:38:57,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:57,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +25: [2023-05-13 09:38:57,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:57,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +18: [2023-05-13 09:38:57,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:57,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:57,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +18: [2023-05-13 09:38:57,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +25: [2023-05-13 09:38:57,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:57,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +25: [2023-05-13 09:38:57,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:57,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:57,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:57,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 7: [2023-05-13 09:38:57,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 7: [2023-05-13 09:38:57,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 7: [2023-05-13 09:38:57,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +16: [2023-05-13 09:38:57,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +10: [2023-05-13 09:38:57,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +10: [2023-05-13 09:38:57,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +17: [2023-05-13 09:38:57,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:57,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:57,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:57,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:57,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:57,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:57,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:57,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:57,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:57,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:57,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:57,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:57,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:57,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:57,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:57,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:57,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:57,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +15: [2023-05-13 09:38:57,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:57,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:57,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:57,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:57,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:57,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:57,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:57,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:57,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:57,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:57,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:57,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:57,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +10: [2023-05-13 09:38:57,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:57,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:57,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:57,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:57,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:57,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:57,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:57,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:57,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:57,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:57,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:57,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +11: [2023-05-13 09:38:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +27: [2023-05-13 09:38:57,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:57,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:57,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:57,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:57,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:57,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:57,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:57,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:57,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:57,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:57,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:57,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +28: [2023-05-13 09:38:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:57,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +11: [2023-05-13 09:38:57,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +24: [2023-05-13 09:38:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:57,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:57,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:57,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:57,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +29: [2023-05-13 09:38:57,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 1: [2023-05-13 09:38:57,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:57,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:57,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:57,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:57,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:57,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:57,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:57,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +27: [2023-05-13 09:38:57,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:57,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:57,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:57,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:57,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +22: [2023-05-13 09:38:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 7: [2023-05-13 09:38:57,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 7: [2023-05-13 09:38:57,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +27: [2023-05-13 09:38:57,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:57,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +25: [2023-05-13 09:38:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +25: [2023-05-13 09:38:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +25: [2023-05-13 09:38:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +25: [2023-05-13 09:38:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +13: [2023-05-13 09:38:57,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:57,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:57,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:57,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:57,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:57,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:57,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +29: [2023-05-13 09:38:57,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:57,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:57,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:57,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +11: [2023-05-13 09:38:57,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:57,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +26: [2023-05-13 09:38:57,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:57,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:57,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:57,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:57,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +11: [2023-05-13 09:38:57,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +25: [2023-05-13 09:38:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:57,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:57,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:57,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:57,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:57,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:57,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +20: [2023-05-13 09:38:57,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:57,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:57,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:57,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:57,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:57,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:57,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:57,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:57,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:57,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +24: [2023-05-13 09:38:57,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:57,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +22: [2023-05-13 09:38:57,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:57,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +11: [2023-05-13 09:38:57,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +11: [2023-05-13 09:38:57,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +15: [2023-05-13 09:38:57,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +15: [2023-05-13 09:38:57,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 0: [2023-05-13 09:38:57,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 0: [2023-05-13 09:38:57,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +28: [2023-05-13 09:38:57,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +11: [2023-05-13 09:38:57,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:57,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +29: [2023-05-13 09:38:57,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:57,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:57,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +22: [2023-05-13 09:38:57,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +29: [2023-05-13 09:38:57,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +13: [2023-05-13 09:38:57,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:57,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:57,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:57,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:57,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:57,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:57,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:57,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:57,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:57,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:57,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:57,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +29: [2023-05-13 09:38:57,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:57,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:57,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:57,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:57,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:57,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +29: [2023-05-13 09:38:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:57,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:57,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:57,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +29: [2023-05-13 09:38:57,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:57,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:57,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:57,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:57,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:57,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:57,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:57,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:57,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 6: [2023-05-13 09:38:57,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 6: [2023-05-13 09:38:57,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:57,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:57,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 4: [2023-05-13 09:38:57,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:57,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +23: [2023-05-13 09:38:57,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +22: [2023-05-13 09:38:57,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:57,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:57,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:57,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:57,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:57,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:57,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:57,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. +30: [2023-05-13 09:38:57,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:57,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_28-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +30: [2023-05-13 09:38:57,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +30: [2023-05-13 09:38:57,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:57,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:57,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:57,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:57,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:57,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:57,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 4: [2023-05-13 09:38:57,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 4: [2023-05-13 09:38:57,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 4: [2023-05-13 09:38:57,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +30: [2023-05-13 09:38:57,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:57,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +12: [2023-05-13 09:38:57,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +12: [2023-05-13 09:38:57,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:57,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:57,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:57,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +12: [2023-05-13 09:38:57,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +12: [2023-05-13 09:38:57,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:57,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +25: [2023-05-13 09:38:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 8: [2023-05-13 09:38:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 8: [2023-05-13 09:38:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:57,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 8: [2023-05-13 09:38:57,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +25: [2023-05-13 09:38:57,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +26: [2023-05-13 09:38:57,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +26: [2023-05-13 09:38:57,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:57,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:57,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:57,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:57,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:57,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +21: [2023-05-13 09:38:57,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:57,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:57,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:57,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:57,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:57,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:57,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:57,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +12: [2023-05-13 09:38:57,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:57,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:57,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +22: [2023-05-13 09:38:57,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:57,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:57,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +30: [2023-05-13 09:38:57,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:57,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:57,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:57,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:57,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:57,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:57,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +31: [2023-05-13 09:38:57,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:57,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +30: [2023-05-13 09:38:57,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:57,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:57,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:57,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +22: [2023-05-13 09:38:57,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:57,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:57,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:57,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:57,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:57,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:57,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:57,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:57,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:57,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:57,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:57,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 1: [2023-05-13 09:38:57,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +30: [2023-05-13 09:38:57,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:57,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 1: [2023-05-13 09:38:57,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:57,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:57,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:57,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:57,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +18: [2023-05-13 09:38:57,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 6: [2023-05-13 09:38:57,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +31: [2023-05-13 09:38:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +31: [2023-05-13 09:38:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:57,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:57,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:57,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:57,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:57,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:57,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:57,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:57,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:57,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:57,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 3: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +23: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt... +21: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:57,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:57,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:57,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:57,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +21: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +21: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:57,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:57,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:57,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:57,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:57,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +27: [2023-05-13 09:38:57,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +27: [2023-05-13 09:38:57,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +20: [2023-05-13 09:38:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +27: [2023-05-13 09:38:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:57,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +20: [2023-05-13 09:38:57,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:57,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:57,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 3: [2023-05-13 09:38:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +19: [2023-05-13 09:38:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:57,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 4: [2023-05-13 09:38:57,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:57,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:57,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 4: [2023-05-13 09:38:57,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +20: [2023-05-13 09:38:57,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +20: [2023-05-13 09:38:57,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +30: [2023-05-13 09:38:57,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:57,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:57,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:57,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +12: [2023-05-13 09:38:57,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:57,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +19: [2023-05-13 09:38:57,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:57,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +26: [2023-05-13 09:38:57,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:57,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +26: [2023-05-13 09:38:57,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:57,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:57,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:57,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:57,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:57,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:57,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... +19: [2023-05-13 09:38:57,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:57,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 4: [2023-05-13 09:38:57,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:57,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +19: [2023-05-13 09:38:57,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:57,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:57,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:57,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:57,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +17: [2023-05-13 09:38:57,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 4: [2023-05-13 09:38:57,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +30: [2023-05-13 09:38:57,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +19: [2023-05-13 09:38:57,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:57,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +19: [2023-05-13 09:38:57,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:57,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:57,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:57,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:57,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:57,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +27: [2023-05-13 09:38:57,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:57,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +30: [2023-05-13 09:38:57,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:57,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:57,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:57,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:57,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +27: [2023-05-13 09:38:57,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:57,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +30: [2023-05-13 09:38:57,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:57,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +18: [2023-05-13 09:38:57,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:57,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:57,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:57,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:57,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:57,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +14: [2023-05-13 09:38:57,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:57,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:57,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:57,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:57,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:57,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:57,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:57,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:57,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:57,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 1: [2023-05-13 09:38:57,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +18: [2023-05-13 09:38:57,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +30: [2023-05-13 09:38:57,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:57,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:57,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 3: [2023-05-13 09:38:57,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:57,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +23: [2023-05-13 09:38:57,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_00-model_states.pt. +30: [2023-05-13 09:38:57,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 1: [2023-05-13 09:38:57,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +14: [2023-05-13 09:38:57,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:57,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +26: [2023-05-13 09:38:57,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:57,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:57,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:57,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:57,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:57,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:57,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +27: [2023-05-13 09:38:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +26: [2023-05-13 09:38:57,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:57,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:57,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:57,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +30: [2023-05-13 09:38:57,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:57,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:57,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:57,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:57,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:57,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:57,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:57,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:57,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +27: [2023-05-13 09:38:57,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +26: [2023-05-13 09:38:57,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:57,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:57,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:57,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:57,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:57,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 6: [2023-05-13 09:38:57,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 6: [2023-05-13 09:38:57,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 3: [2023-05-13 09:38:57,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +28: [2023-05-13 09:38:57,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +28: [2023-05-13 09:38:57,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +28: [2023-05-13 09:38:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +28: [2023-05-13 09:38:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +10: [2023-05-13 09:38:57,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +10: [2023-05-13 09:38:57,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:57,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:57,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:57,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +24: [2023-05-13 09:38:57,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +10: [2023-05-13 09:38:57,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:57,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +14: [2023-05-13 09:38:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +21: [2023-05-13 09:38:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +13: [2023-05-13 09:38:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +26: [2023-05-13 09:38:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 3: [2023-05-13 09:38:57,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +21: [2023-05-13 09:38:57,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:57,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:57,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +21: [2023-05-13 09:38:57,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:57,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:57,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:57,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +14: [2023-05-13 09:38:57,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:57,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:57,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:57,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:57,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:57,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +24: [2023-05-13 09:38:57,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:57,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:57,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +24: [2023-05-13 09:38:57,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:57,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +12: [2023-05-13 09:38:57,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:57,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:57,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:57,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:57,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +31: [2023-05-13 09:38:57,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +10: [2023-05-13 09:38:57,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +11: [2023-05-13 09:38:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +31: [2023-05-13 09:38:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +11: [2023-05-13 09:38:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 9: [2023-05-13 09:38:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:57,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:57,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:57,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 9: [2023-05-13 09:38:57,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:57,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:57,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:57,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:57,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +21: [2023-05-13 09:38:57,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:57,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +11: [2023-05-13 09:38:57,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:57,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +21: [2023-05-13 09:38:57,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:57,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +16: [2023-05-13 09:38:57,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +11: [2023-05-13 09:38:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:57,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +11: [2023-05-13 09:38:57,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:57,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +28: [2023-05-13 09:38:57,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +16: [2023-05-13 09:38:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +16: [2023-05-13 09:38:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:57,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:57,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:57,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 6: [2023-05-13 09:38:57,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +16: [2023-05-13 09:38:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +21: [2023-05-13 09:38:57,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:57,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +17: [2023-05-13 09:38:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:57,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:57,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:57,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:57,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:57,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:57,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:57,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:57,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:57,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:57,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:57,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +12: [2023-05-13 09:38:57,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +24: [2023-05-13 09:38:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:57,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +29: [2023-05-13 09:38:57,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:57,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:57,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +24: [2023-05-13 09:38:57,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:57,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:57,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:57,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:57,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:57,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:57,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:57,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:57,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:57,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:57,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +10: [2023-05-13 09:38:57,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:57,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +13: [2023-05-13 09:38:57,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +13: [2023-05-13 09:38:57,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:57,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:57,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +13: [2023-05-13 09:38:57,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:57,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:57,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 7: [2023-05-13 09:38:57,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:57,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:57,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +12: [2023-05-13 09:38:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +11: [2023-05-13 09:38:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +11: [2023-05-13 09:38:57,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +17: [2023-05-13 09:38:57,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +11: [2023-05-13 09:38:57,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +29: [2023-05-13 09:38:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:57,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:57,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:57,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:57,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:57,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:57,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:57,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:57,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:57,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:57,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:57,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:57,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:57,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +28: [2023-05-13 09:38:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:57,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:57,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:57,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:57,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:57,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:57,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:57,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:57,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +17: [2023-05-13 09:38:57,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:57,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:57,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:57,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:57,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 9: [2023-05-13 09:38:57,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +17: [2023-05-13 09:38:57,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:57,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:57,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +15: [2023-05-13 09:38:57,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +15: [2023-05-13 09:38:57,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:57,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:57,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:57,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:57,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:57,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:57,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:57,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:57,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:57,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:57,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:57,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:57,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:57,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:57,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:57,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:57,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:57,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +28: [2023-05-13 09:38:57,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +28: [2023-05-13 09:38:57,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:57,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:57,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:57,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:57,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:57,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:57,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:57,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +18: [2023-05-13 09:38:57,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:57,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:57,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:57,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +23: [2023-05-13 09:38:57,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_29-model_01-model_states.pt. +22: [2023-05-13 09:38:57,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:57,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:57,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:57,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:57,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:57,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:57,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:57,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:57,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:57,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:57,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:57,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:57,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:57,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:57,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:57,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:57,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:57,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:57,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:57,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:57,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +18: [2023-05-13 09:38:57,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +23: [2023-05-13 09:38:57,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt... +31: [2023-05-13 09:38:57,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +31: [2023-05-13 09:38:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:57,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:57,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +10: [2023-05-13 09:38:57,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:57,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:57,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:57,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:57,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:57,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +26: [2023-05-13 09:38:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +26: [2023-05-13 09:38:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +26: [2023-05-13 09:38:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... + 7: [2023-05-13 09:38:57,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 7: [2023-05-13 09:38:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 7: [2023-05-13 09:38:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 7: [2023-05-13 09:38:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:57,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:57,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +25: [2023-05-13 09:38:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:57,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:57,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +21: [2023-05-13 09:38:57,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +21: [2023-05-13 09:38:57,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +21: [2023-05-13 09:38:57,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +21: [2023-05-13 09:38:57,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +22: [2023-05-13 09:38:57,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:57,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:57,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:57,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +15: [2023-05-13 09:38:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +15: [2023-05-13 09:38:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +18: [2023-05-13 09:38:57,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +26: [2023-05-13 09:38:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +15: [2023-05-13 09:38:57,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +22: [2023-05-13 09:38:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:57,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:57,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:57,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:57,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:57,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:57,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:57,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:57,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +22: [2023-05-13 09:38:57,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 7: [2023-05-13 09:38:57,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 8: [2023-05-13 09:38:57,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 0: [2023-05-13 09:38:57,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 0: [2023-05-13 09:38:57,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 0: [2023-05-13 09:38:57,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +22: [2023-05-13 09:38:57,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 0: [2023-05-13 09:38:57,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +22: [2023-05-13 09:38:57,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:57,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:57,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:57,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:57,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:57,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:57,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +30: [2023-05-13 09:38:57,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +30: [2023-05-13 09:38:57,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +30: [2023-05-13 09:38:57,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:57,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +29: [2023-05-13 09:38:57,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:57,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:57,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:57,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:57,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:57,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:57,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:57,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:57,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +29: [2023-05-13 09:38:57,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 7: [2023-05-13 09:38:57,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:57,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +21: [2023-05-13 09:38:57,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:57,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:57,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:57,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +23: [2023-05-13 09:38:57,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_00-model_states.pt. +11: [2023-05-13 09:38:57,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:57,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:57,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +23: [2023-05-13 09:38:57,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +23: [2023-05-13 09:38:57,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt... +22: [2023-05-13 09:38:57,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +15: [2023-05-13 09:38:57,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:57,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +22: [2023-05-13 09:38:57,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:57,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:57,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +22: [2023-05-13 09:38:57,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:57,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:57,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:57,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:57,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:57,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:57,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:57,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:57,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:57,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:57,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:57,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:57,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:57,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:57,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:57,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:57,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:57,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:57,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:57,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:57,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:57,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:57,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:57,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:57,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:57,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:57,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:57,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:57,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:57,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:57,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +27: [2023-05-13 09:38:57,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:57,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:57,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:57,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:57,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:57,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:57,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:57,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:57,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:57,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:57,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:57,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +16: [2023-05-13 09:38:57,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:57,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:57,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:57,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:57,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:57,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:57,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:57,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:57,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:57,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:57,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:57,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:57,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:57,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:57,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:57,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:57,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:57,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:57,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:57,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:57,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:57,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:57,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:57,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:57,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +27: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +27: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +22: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +26: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +27: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:57,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:57,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:57,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +15: [2023-05-13 09:38:57,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 7: [2023-05-13 09:38:57,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +15: [2023-05-13 09:38:57,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +15: [2023-05-13 09:38:57,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:57,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:57,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +22: [2023-05-13 09:38:57,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:57,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:57,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:57,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:57,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:57,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:57,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:57,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:57,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +22: [2023-05-13 09:38:57,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:57,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:57,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +29: [2023-05-13 09:38:57,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:57,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:57,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:57,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:57,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:57,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +29: [2023-05-13 09:38:57,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +29: [2023-05-13 09:38:57,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:57,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:57,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:57,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:57,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:57,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +27: [2023-05-13 09:38:57,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:57,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:57,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:57,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:57,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:57,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:57,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:57,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:57,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:57,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +30: [2023-05-13 09:38:57,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:57,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:57,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:57,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:57,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:57,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:57,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:57,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:57,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:57,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:57,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:57,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:57,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:57,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:57,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:57,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:57,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +30: [2023-05-13 09:38:57,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:57,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +14: [2023-05-13 09:38:57,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 6: [2023-05-13 09:38:57,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 6: [2023-05-13 09:38:57,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +19: [2023-05-13 09:38:57,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:57,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:57,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 6: [2023-05-13 09:38:57,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +14: [2023-05-13 09:38:57,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +14: [2023-05-13 09:38:57,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +20: [2023-05-13 09:38:57,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:57,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +30: [2023-05-13 09:38:57,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:57,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:57,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:57,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:57,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:57,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:57,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +30: [2023-05-13 09:38:57,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:57,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:57,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:57,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:57,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:57,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:57,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:57,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:57,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:57,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:57,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:57,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:57,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +17: [2023-05-13 09:38:57,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +17: [2023-05-13 09:38:57,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +17: [2023-05-13 09:38:57,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +14: [2023-05-13 09:38:57,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:57,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:57,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:57,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:57,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:57,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:57,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:57,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:57,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:57,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:57,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +30: [2023-05-13 09:38:57,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:57,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:57,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 1: [2023-05-13 09:38:57,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 1: [2023-05-13 09:38:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:57,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 8: [2023-05-13 09:38:57,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:57,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:57,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +24: [2023-05-13 09:38:57,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +24: [2023-05-13 09:38:57,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:57,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:57,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:57,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:57,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:57,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:57,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:57,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:57,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:57,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:57,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 9: [2023-05-13 09:38:57,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +17: [2023-05-13 09:38:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:57,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:57,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:57,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:57,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:57,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:57,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +16: [2023-05-13 09:38:57,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:57,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +27: [2023-05-13 09:38:57,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:57,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:57,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +12: [2023-05-13 09:38:57,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +12: [2023-05-13 09:38:57,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +12: [2023-05-13 09:38:57,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +12: [2023-05-13 09:38:57,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +10: [2023-05-13 09:38:57,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +10: [2023-05-13 09:38:57,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +10: [2023-05-13 09:38:57,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +10: [2023-05-13 09:38:57,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 4: [2023-05-13 09:38:57,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:57,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +26: [2023-05-13 09:38:57,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:57,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:57,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:57,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 4: [2023-05-13 09:38:57,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +27: [2023-05-13 09:38:57,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +20: [2023-05-13 09:38:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +24: [2023-05-13 09:38:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 3: [2023-05-13 09:38:57,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +20: [2023-05-13 09:38:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +20: [2023-05-13 09:38:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +16: [2023-05-13 09:38:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +16: [2023-05-13 09:38:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +26: [2023-05-13 09:38:57,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:57,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:57,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +26: [2023-05-13 09:38:57,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:57,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:57,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:57,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +14: [2023-05-13 09:38:57,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:57,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:57,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:57,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +14: [2023-05-13 09:38:57,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:57,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +10: [2023-05-13 09:38:57,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:57,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:57,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +12: [2023-05-13 09:38:57,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:57,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:57,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:57,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:57,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +12: [2023-05-13 09:38:57,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +17: [2023-05-13 09:38:57,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:57,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +25: [2023-05-13 09:38:57,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:57,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +31: [2023-05-13 09:38:57,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:57,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:57,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +31: [2023-05-13 09:38:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 1: [2023-05-13 09:38:57,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:57,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:57,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:57,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:57,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:57,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:57,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:57,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:57,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +28: [2023-05-13 09:38:57,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +25: [2023-05-13 09:38:57,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:57,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:57,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:57,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +17: [2023-05-13 09:38:57,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:57,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:57,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:57,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 6: [2023-05-13 09:38:57,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +12: [2023-05-13 09:38:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:57,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:57,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +11: [2023-05-13 09:38:57,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:57,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:57,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +31: [2023-05-13 09:38:57,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +25: [2023-05-13 09:38:57,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +17: [2023-05-13 09:38:57,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +19: [2023-05-13 09:38:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +19: [2023-05-13 09:38:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 0: [2023-05-13 09:38:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:57,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +31: [2023-05-13 09:38:57,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:57,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +16: [2023-05-13 09:38:57,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:57,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:57,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:57,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:57,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:57,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +18: [2023-05-13 09:38:57,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 9: [2023-05-13 09:38:57,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +16: [2023-05-13 09:38:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +24: [2023-05-13 09:38:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +10: [2023-05-13 09:38:57,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:57,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +16: [2023-05-13 09:38:57,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +25: [2023-05-13 09:38:57,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:57,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:57,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +24: [2023-05-13 09:38:57,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:57,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 1: [2023-05-13 09:38:57,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +24: [2023-05-13 09:38:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 6: [2023-05-13 09:38:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +23: [2023-05-13 09:38:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:57,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +19: [2023-05-13 09:38:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +10: [2023-05-13 09:38:57,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:57,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:57,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +12: [2023-05-13 09:38:57,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +31: [2023-05-13 09:38:57,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:57,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:57,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +18: [2023-05-13 09:38:57,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 5: [2023-05-13 09:38:57,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 3: [2023-05-13 09:38:57,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:57,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:57,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +23: [2023-05-13 09:38:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:57,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +18: [2023-05-13 09:38:57,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 0: [2023-05-13 09:38:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 0: [2023-05-13 09:38:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:57,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:57,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:57,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:57,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +11: [2023-05-13 09:38:57,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:57,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +13: [2023-05-13 09:38:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +13: [2023-05-13 09:38:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +13: [2023-05-13 09:38:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. + 2: [2023-05-13 09:38:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +13: [2023-05-13 09:38:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +11: [2023-05-13 09:38:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +11: [2023-05-13 09:38:57,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:57,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +28: [2023-05-13 09:38:57,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +12: [2023-05-13 09:38:57,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:57,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:57,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:57,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +31: [2023-05-13 09:38:57,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +12: [2023-05-13 09:38:57,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:57,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:57,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:57,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +31: [2023-05-13 09:38:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +28: [2023-05-13 09:38:57,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:57,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:57,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 3: [2023-05-13 09:38:57,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:57,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:57,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:57,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:57,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +23: [2023-05-13 09:38:57,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_30-model_01-model_states.pt. +13: [2023-05-13 09:38:57,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +13: [2023-05-13 09:38:57,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +18: [2023-05-13 09:38:57,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +19: [2023-05-13 09:38:57,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +19: [2023-05-13 09:38:57,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:57,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:57,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 9: [2023-05-13 09:38:57,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:57,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:57,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:57,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:57,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:57,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:57,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:57,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +28: [2023-05-13 09:38:57,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:57,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:57,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:57,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:57,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:57,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +18: [2023-05-13 09:38:57,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:57,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:57,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:57,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:57,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:57,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:57,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 9: [2023-05-13 09:38:57,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:57,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:57,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:57,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:57,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:57,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:57,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:57,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:57,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:57,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:57,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +18: [2023-05-13 09:38:57,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +15: [2023-05-13 09:38:57,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:57,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:57,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:57,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:57,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:57,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:57,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:57,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:57,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:57,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:57,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:57,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:57,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:57,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:57,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:57,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:57,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +23: [2023-05-13 09:38:57,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt... +29: [2023-05-13 09:38:57,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:57,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:57,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:57,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:57,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 5: [2023-05-13 09:38:57,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:57,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. + 3: [2023-05-13 09:38:57,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:57,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... + 2: [2023-05-13 09:38:57,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:57,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:57,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +21: [2023-05-13 09:38:57,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:57,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +14: [2023-05-13 09:38:57,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:57,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:57,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:57,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +21: [2023-05-13 09:38:57,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:57,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:57,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:57,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +13: [2023-05-13 09:38:57,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +13: [2023-05-13 09:38:57,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +14: [2023-05-13 09:38:57,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:57,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:57,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:57,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:57,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:57,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:57,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:57,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:57,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:57,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:57,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:57,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +23: [2023-05-13 09:38:57,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +29: [2023-05-13 09:38:57,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:57,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +23: [2023-05-13 09:38:57,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_00-model_states.pt. +13: [2023-05-13 09:38:57,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +23: [2023-05-13 09:38:57,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +29: [2023-05-13 09:38:57,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:57,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:57,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt... +29: [2023-05-13 09:38:57,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:57,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:57,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:57,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:57,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:57,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:57,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:57,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:57,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:57,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:57,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:57,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:57,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:57,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:57,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:57,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:57,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:57,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:57,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:57,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:57,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:57,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:57,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:57,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:57,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:57,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:57,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:57,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:57,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:57,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:57,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:57,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:57,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:57,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:57,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:57,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:57,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:57,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:57,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:57,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:57,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:57,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:57,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:57,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:57,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:57,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:57,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:57,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:57,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:57,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:57,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:57,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:57,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:57,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:57,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:57,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:57,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:57,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:57,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:57,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:57,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:57,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:57,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:57,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:57,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:57,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:57,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:57,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:57,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:57,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:57,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:57,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:57,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:57,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:57,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:57,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:57,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:57,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:57,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:57,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:57,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:57,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:57,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:57,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:57,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:57,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:57,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:57,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:57,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:57,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:57,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:57,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:57,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:57,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:57,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:57,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:57,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:57,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:57,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:57,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:57,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:57,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:57,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:57,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:57,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 7: [2023-05-13 09:38:57,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:57,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:57,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:57,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:57,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:57,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:57,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:57,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:57,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:57,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:57,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:57,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:57,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:57,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:57,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:57,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:57,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:57,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:57,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:57,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:57,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:57,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:57,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:58,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:58,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:58,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:58,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:58,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:58,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +15: [2023-05-13 09:38:58,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:58,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:58,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:58,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:58,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:58,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 8: [2023-05-13 09:38:58,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +15: [2023-05-13 09:38:58,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 8: [2023-05-13 09:38:58,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:58,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:58,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:58,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:58,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:58,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:58,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:58,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:58,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:58,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +27: [2023-05-13 09:38:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:58,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:58,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:58,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:58,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:58,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:58,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:58,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:58,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:58,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:58,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:58,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:58,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:58,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:58,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:58,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:58,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:58,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:58,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:58,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:58,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:58,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:58,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:58,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:58,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:58,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:58,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:58,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:58,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:58,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:58,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:58,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:58,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:58,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:58,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:58,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:58,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:58,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:58,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:58,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:58,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:58,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:58,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:58,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:58,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:58,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:58,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:58,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:58,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:58,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:58,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:58,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:58,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:58,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 8: [2023-05-13 09:38:58,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:58,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:58,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:58,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:58,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:58,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 8: [2023-05-13 09:38:58,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +15: [2023-05-13 09:38:58,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:58,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:58,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 8: [2023-05-13 09:38:58,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:58,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:58,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:58,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 8: [2023-05-13 09:38:58,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:58,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:58,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:58,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:58,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:58,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:58,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:58,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:58,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:58,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +15: [2023-05-13 09:38:58,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:58,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:58,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:58,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:58,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:58,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 8: [2023-05-13 09:38:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:58,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 8: [2023-05-13 09:38:58,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +29: [2023-05-13 09:38:58,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:58,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:58,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:58,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:58,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +21: [2023-05-13 09:38:58,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +26: [2023-05-13 09:38:58,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:58,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:58,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:58,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:58,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +22: [2023-05-13 09:38:58,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:58,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:58,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:58,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 4: [2023-05-13 09:38:58,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:58,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:58,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:58,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +25: [2023-05-13 09:38:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +25: [2023-05-13 09:38:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 1: [2023-05-13 09:38:58,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:58,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:58,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:58,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:58,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:58,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:58,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:58,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:58,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:58,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:58,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:58,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:58,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:58,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 8: [2023-05-13 09:38:58,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:58,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +25: [2023-05-13 09:38:58,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:58,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +29: [2023-05-13 09:38:58,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 8: [2023-05-13 09:38:58,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:58,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:58,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:58,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:58,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:58,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:58,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +29: [2023-05-13 09:38:58,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:58,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:58,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:58,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +29: [2023-05-13 09:38:58,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:58,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:58,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:58,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:58,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:58,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +12: [2023-05-13 09:38:58,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:58,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +22: [2023-05-13 09:38:58,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:58,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:58,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 1: [2023-05-13 09:38:58,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:58,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:58,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:58,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +16: [2023-05-13 09:38:58,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +22: [2023-05-13 09:38:58,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:58,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:58,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +22: [2023-05-13 09:38:58,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:58,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:58,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:58,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:58,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:58,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:58,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:58,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:58,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:58,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:58,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:58,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:58,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:58,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 4: [2023-05-13 09:38:58,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:58,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:58,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:58,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:58,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:58,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:58,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:58,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:58,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +25: [2023-05-13 09:38:58,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +25: [2023-05-13 09:38:58,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:58,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:58,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +29: [2023-05-13 09:38:58,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:58,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:58,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:58,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:58,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:58,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:58,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:58,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:58,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:58,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +12: [2023-05-13 09:38:58,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:58,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:58,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:58,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +20: [2023-05-13 09:38:58,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +27: [2023-05-13 09:38:58,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +27: [2023-05-13 09:38:58,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +27: [2023-05-13 09:38:58,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +27: [2023-05-13 09:38:58,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +20: [2023-05-13 09:38:58,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +20: [2023-05-13 09:38:58,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +12: [2023-05-13 09:38:58,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:58,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +12: [2023-05-13 09:38:58,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:58,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +16: [2023-05-13 09:38:58,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:58,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +16: [2023-05-13 09:38:58,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 6: [2023-05-13 09:38:58,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:58,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:58,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +30: [2023-05-13 09:38:58,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +24: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +24: [2023-05-13 09:38:58,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:58,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:58,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:58,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:58,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:58,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:58,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:58,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +14: [2023-05-13 09:38:58,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:58,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:58,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:58,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 2: [2023-05-13 09:38:58,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 2: [2023-05-13 09:38:58,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 2: [2023-05-13 09:38:58,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +16: [2023-05-13 09:38:58,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:58,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:58,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:58,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:58,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +17: [2023-05-13 09:38:58,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +17: [2023-05-13 09:38:58,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +27: [2023-05-13 09:38:58,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:58,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:58,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:58,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:58,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:58,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:58,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:58,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +20: [2023-05-13 09:38:58,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:58,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:58,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:58,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:58,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:58,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:58,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:58,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:58,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:58,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +10: [2023-05-13 09:38:58,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:58,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:58,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:58,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:58,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:58,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:58,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:58,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:58,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:58,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +17: [2023-05-13 09:38:58,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:58,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:58,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:58,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +24: [2023-05-13 09:38:58,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:58,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:58,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +11: [2023-05-13 09:38:58,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 9: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 9: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +28: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +26: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +29: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 2: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +30: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:58,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:58,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +24: [2023-05-13 09:38:58,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:58,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +26: [2023-05-13 09:38:58,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:58,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +15: [2023-05-13 09:38:58,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:58,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:58,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:58,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:58,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:58,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:58,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:58,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +19: [2023-05-13 09:38:58,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +21: [2023-05-13 09:38:58,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:58,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:58,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +19: [2023-05-13 09:38:58,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:58,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:58,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +31: [2023-05-13 09:38:58,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:58,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:58,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:58,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:58,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:58,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +23: [2023-05-13 09:38:58,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 9: [2023-05-13 09:38:58,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:58,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +29: [2023-05-13 09:38:58,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:58,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:58,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:58,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:58,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:58,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:58,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:58,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +29: [2023-05-13 09:38:58,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:58,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:58,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:58,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +14: [2023-05-13 09:38:58,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:58,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:58,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:58,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:58,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:58,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:58,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:58,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +27: [2023-05-13 09:38:58,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:58,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:58,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:58,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +20: [2023-05-13 09:38:58,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:58,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +20: [2023-05-13 09:38:58,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:58,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +10: [2023-05-13 09:38:58,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:58,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 6: [2023-05-13 09:38:58,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:58,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +10: [2023-05-13 09:38:58,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:58,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:58,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:58,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:58,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 6: [2023-05-13 09:38:58,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:58,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +19: [2023-05-13 09:38:58,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +27: [2023-05-13 09:38:58,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:58,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:58,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +13: [2023-05-13 09:38:58,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:58,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:58,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:58,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:58,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:58,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +26: [2023-05-13 09:38:58,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:58,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 2: [2023-05-13 09:38:58,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:58,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +28: [2023-05-13 09:38:58,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +28: [2023-05-13 09:38:58,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +10: [2023-05-13 09:38:58,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +27: [2023-05-13 09:38:58,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +27: [2023-05-13 09:38:58,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +23: [2023-05-13 09:38:58,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:58,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:58,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:58,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:58,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +20: [2023-05-13 09:38:58,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +10: [2023-05-13 09:38:58,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +14: [2023-05-13 09:38:58,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 3: [2023-05-13 09:38:58,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 2: [2023-05-13 09:38:58,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +10: [2023-05-13 09:38:58,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +26: [2023-05-13 09:38:58,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +26: [2023-05-13 09:38:58,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +20: [2023-05-13 09:38:58,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:58,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +10: [2023-05-13 09:38:58,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +15: [2023-05-13 09:38:58,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:58,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:58,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 2: [2023-05-13 09:38:58,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +26: [2023-05-13 09:38:58,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:58,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +26: [2023-05-13 09:38:58,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +14: [2023-05-13 09:38:58,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:58,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +15: [2023-05-13 09:38:58,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:58,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:58,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 2: [2023-05-13 09:38:58,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +13: [2023-05-13 09:38:58,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:58,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:58,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:58,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:58,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:58,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:58,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:58,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:58,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:58,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:58,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:58,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:58,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:58,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +30: [2023-05-13 09:38:58,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +15: [2023-05-13 09:38:58,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:58,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:58,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:58,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 9: [2023-05-13 09:38:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +30: [2023-05-13 09:38:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:58,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:58,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:58,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +21: [2023-05-13 09:38:58,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:58,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:58,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:58,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +31: [2023-05-13 09:38:58,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:58,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +24: [2023-05-13 09:38:58,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:58,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:58,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:58,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:58,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +11: [2023-05-13 09:38:58,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:58,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:58,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:58,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:58,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:58,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:58,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:58,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:58,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:58,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +11: [2023-05-13 09:38:58,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +24: [2023-05-13 09:38:58,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +15: [2023-05-13 09:38:58,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:58,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:58,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +11: [2023-05-13 09:38:58,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:58,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 5: [2023-05-13 09:38:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 5: [2023-05-13 09:38:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 5: [2023-05-13 09:38:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 7: [2023-05-13 09:38:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:58,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +13: [2023-05-13 09:38:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +13: [2023-05-13 09:38:58,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. + 9: [2023-05-13 09:38:58,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +31: [2023-05-13 09:38:58,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:58,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:58,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:58,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:58,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +18: [2023-05-13 09:38:58,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +18: [2023-05-13 09:38:58,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +19: [2023-05-13 09:38:58,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:58,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_31-model_01-model_states.pt. +15: [2023-05-13 09:38:58,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:58,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 9: [2023-05-13 09:38:58,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:58,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:58,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:58,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +15: [2023-05-13 09:38:58,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +21: [2023-05-13 09:38:58,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:58,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:58,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:58,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:58,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +19: [2023-05-13 09:38:58,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +21: [2023-05-13 09:38:58,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:58,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:58,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +19: [2023-05-13 09:38:58,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +23: [2023-05-13 09:38:58,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +23: [2023-05-13 09:38:58,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:58,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +23: [2023-05-13 09:38:58,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 7: [2023-05-13 09:38:58,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:58,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:58,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:58,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +23: [2023-05-13 09:38:58,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +13: [2023-05-13 09:38:58,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +17: [2023-05-13 09:38:58,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:58,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +18: [2023-05-13 09:38:58,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:58,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:58,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +13: [2023-05-13 09:38:58,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt... +28: [2023-05-13 09:38:58,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:58,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:58,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:58,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 3: [2023-05-13 09:38:58,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +28: [2023-05-13 09:38:58,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:58,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:58,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 3: [2023-05-13 09:38:58,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:58,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:58,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 3: [2023-05-13 09:38:58,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +28: [2023-05-13 09:38:58,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:58,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:58,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:58,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +17: [2023-05-13 09:38:58,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:58,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:58,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:58,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:58,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:58,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:58,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:58,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:58,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:58,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:58,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:58,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 5: [2023-05-13 09:38:58,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +18: [2023-05-13 09:38:58,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +18: [2023-05-13 09:38:58,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +17: [2023-05-13 09:38:58,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:58,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. +13: [2023-05-13 09:38:58,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:58,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:58,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:58,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 0: [2023-05-13 09:38:58,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 5: [2023-05-13 09:38:58,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 5: [2023-05-13 09:38:58,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... + 5: [2023-05-13 09:38:58,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt... +27: [2023-05-13 09:38:58,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:58,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:58,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:58,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:58,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:58,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:58,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:58,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:58,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:58,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:58,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:58,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:58,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:58,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:58,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:58,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:58,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:58,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:58,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:58,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:58,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:58,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:58,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +15: [2023-05-13 09:38:58,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +15: [2023-05-13 09:38:58,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:58,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:58,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:58,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:58,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:58,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:58,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:58,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:58,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:58,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:58,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 8: [2023-05-13 09:38:58,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 8: [2023-05-13 09:38:58,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:58,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:58,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:58,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:58,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:58,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:58,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:58,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:58,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:58,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 8: [2023-05-13 09:38:58,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:58,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +20: [2023-05-13 09:38:58,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:58,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:58,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:58,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:58,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:58,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:58,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +20: [2023-05-13 09:38:58,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:58,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:58,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:58,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:58,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:58,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:58,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:58,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:58,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:58,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:58,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:58,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +29: [2023-05-13 09:38:58,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:58,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:58,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:58,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:58,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:58,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:58,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:58,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:58,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:58,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:58,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:58,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:58,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 1: [2023-05-13 09:38:58,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:58,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:58,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:58,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:58,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:58,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:58,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 8: [2023-05-13 09:38:58,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +15: [2023-05-13 09:38:58,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:58,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:58,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:58,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:58,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:58,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:58,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:58,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:58,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +17: [2023-05-13 09:38:58,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:58,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 8: [2023-05-13 09:38:58,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:58,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 8: [2023-05-13 09:38:58,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:58,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:58,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:58,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:58,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:58,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:58,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:58,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:58,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:58,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:58,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:58,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:58,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +19: [2023-05-13 09:38:58,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:58,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:58,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:58,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:58,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:58,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:58,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:58,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:58,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:58,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:58,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:58,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +20: [2023-05-13 09:38:58,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 1: [2023-05-13 09:38:58,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +23: [2023-05-13 09:38:58,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:58,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:58,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:58,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:58,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:58,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:58,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:58,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:58,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:58,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +22: [2023-05-13 09:38:58,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +22: [2023-05-13 09:38:58,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:58,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:58,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +20: [2023-05-13 09:38:58,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:58,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:58,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:58,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:58,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 1: [2023-05-13 09:38:58,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +25: [2023-05-13 09:38:58,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:58,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:58,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:58,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:58,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:58,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:58,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:58,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +17: [2023-05-13 09:38:58,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:58,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:58,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:58,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:58,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:58,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:58,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:58,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +12: [2023-05-13 09:38:58,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:58,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:58,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:58,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:58,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:58,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:58,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:58,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:58,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:58,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:58,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:58,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:58,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:58,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:58,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:58,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +23: [2023-05-13 09:38:58,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:58,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:58,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:58,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:58,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:58,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:58,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:58,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:58,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:58,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:58,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:58,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:58,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:58,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:58,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:58,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:58,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:58,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:58,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:58,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:58,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:58,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:58,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:58,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +11: [2023-05-13 09:38:58,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:58,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:58,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:58,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:58,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +22: [2023-05-13 09:38:58,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:58,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +19: [2023-05-13 09:38:58,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:58,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:58,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:58,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:58,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:58,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +19: [2023-05-13 09:38:58,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:58,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:58,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:58,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:58,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:58,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:58,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:58,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:58,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:58,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:58,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +10: [2023-05-13 09:38:58,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:58,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:58,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +10: [2023-05-13 09:38:58,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +25: [2023-05-13 09:38:58,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:58,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:58,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +12: [2023-05-13 09:38:58,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +12: [2023-05-13 09:38:58,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:58,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:58,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:58,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:58,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:58,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:58,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:58,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:58,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:58,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:58,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:58,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:58,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:58,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:58,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:58,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:58,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:58,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:58,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:58,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +17: [2023-05-13 09:38:58,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:58,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:58,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:58,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:58,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:58,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:58,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:58,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:58,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:58,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:58,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:58,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:58,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:58,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:58,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:58,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:58,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:58,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:58,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:58,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:58,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:58,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:58,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:58,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:58,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:58,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:58,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +19: [2023-05-13 09:38:58,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +19: [2023-05-13 09:38:58,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +19: [2023-05-13 09:38:58,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +31: [2023-05-13 09:38:58,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +31: [2023-05-13 09:38:58,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +31: [2023-05-13 09:38:58,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +18: [2023-05-13 09:38:58,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +18: [2023-05-13 09:38:58,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +18: [2023-05-13 09:38:58,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +13: [2023-05-13 09:38:58,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +13: [2023-05-13 09:38:58,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +13: [2023-05-13 09:38:58,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +18: [2023-05-13 09:38:58,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +13: [2023-05-13 09:38:58,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +31: [2023-05-13 09:38:58,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:58,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +26: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +26: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +28: [2023-05-13 09:38:58,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +28: [2023-05-13 09:38:58,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +24: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +16: [2023-05-13 09:38:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 7: [2023-05-13 09:38:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 7: [2023-05-13 09:38:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 7: [2023-05-13 09:38:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:58,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +21: [2023-05-13 09:38:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 7: [2023-05-13 09:38:58,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +21: [2023-05-13 09:38:58,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:58,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:58,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:58,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +12: [2023-05-13 09:38:58,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:58,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:58,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:58,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +30: [2023-05-13 09:38:58,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:58,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:58,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:58,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:58,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:58,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:58,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:58,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +28: [2023-05-13 09:38:58,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +19: [2023-05-13 09:38:58,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:58,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:58,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:58,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:58,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:58,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:58,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:58,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:58,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:58,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:58,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:58,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +29: [2023-05-13 09:38:58,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:58,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:58,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:58,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:58,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +25: [2023-05-13 09:38:58,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +25: [2023-05-13 09:38:58,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 4: [2023-05-13 09:38:58,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +25: [2023-05-13 09:38:58,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +29: [2023-05-13 09:38:58,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:58,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:58,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:58,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:58,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:58,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:58,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:58,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:58,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:58,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:58,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:58,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:58,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +13: [2023-05-13 09:38:58,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:58,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:58,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:58,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:58,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +24: [2023-05-13 09:38:58,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:58,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:58,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:58,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:58,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +12: [2023-05-13 09:38:58,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:58,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:58,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:58,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:58,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:58,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:58,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:58,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +19: [2023-05-13 09:38:58,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:58,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +28: [2023-05-13 09:38:58,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:58,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:58,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +25: [2023-05-13 09:38:58,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +29: [2023-05-13 09:38:58,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:58,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:58,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:58,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:58,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +27: [2023-05-13 09:38:58,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +27: [2023-05-13 09:38:58,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 9: [2023-05-13 09:38:58,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:58,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +26: [2023-05-13 09:38:58,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 8: [2023-05-13 09:38:58,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:58,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:58,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 3: [2023-05-13 09:38:58,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 3: [2023-05-13 09:38:58,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +19: [2023-05-13 09:38:58,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:58,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 1: [2023-05-13 09:38:58,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:58,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:58,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 2: [2023-05-13 09:38:58,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:58,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:58,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +14: [2023-05-13 09:38:58,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:58,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:58,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:58,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +27: [2023-05-13 09:38:58,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:58,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:58,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 2: [2023-05-13 09:38:58,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:58,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:58,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:58,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:58,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:58,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:58,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:58,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:58,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:58,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 0: [2023-05-13 09:38:58,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:58,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:58,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +19: [2023-05-13 09:38:58,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +28: [2023-05-13 09:38:58,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:58,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:58,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:58,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:58,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +27: [2023-05-13 09:38:58,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:58,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 6: [2023-05-13 09:38:58,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:58,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:58,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:58,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:58,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:58,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:58,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:58,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:58,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:58,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:58,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:58,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:58,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:58,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:58,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:58,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 9: [2023-05-13 09:38:58,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:58,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:58,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:58,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:58,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +21: [2023-05-13 09:38:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:58,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:58,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +28: [2023-05-13 09:38:58,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:58,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:58,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:58,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 2: [2023-05-13 09:38:58,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +13: [2023-05-13 09:38:58,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:58,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +30: [2023-05-13 09:38:58,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:58,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +25: [2023-05-13 09:38:58,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:58,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:58,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +26: [2023-05-13 09:38:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:58,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:58,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:58,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:58,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:58,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:58,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:58,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:58,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 9: [2023-05-13 09:38:58,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:58,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:58,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:58,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:58,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +21: [2023-05-13 09:38:58,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:58,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:58,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:58,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:58,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:58,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +28: [2023-05-13 09:38:58,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:58,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:58,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:58,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:58,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:58,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:58,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:58,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:58,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:58,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +16: [2023-05-13 09:38:58,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:58,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:58,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +25: [2023-05-13 09:38:58,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:58,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:58,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +30: [2023-05-13 09:38:58,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:58,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 9: [2023-05-13 09:38:58,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:58,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:58,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:58,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +25: [2023-05-13 09:38:58,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:58,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 9: [2023-05-13 09:38:58,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +25: [2023-05-13 09:38:58,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +14: [2023-05-13 09:38:58,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:58,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:58,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +14: [2023-05-13 09:38:58,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +21: [2023-05-13 09:38:58,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:58,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:58,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:58,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:58,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:58,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:58,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:58,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:58,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:58,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:58,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:58,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:58,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:58,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:58,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:58,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +28: [2023-05-13 09:38:58,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:58,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:58,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:58,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:58,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:58,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:58,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +23: [2023-05-13 09:38:58,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:58,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:58,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:58,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:58,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +28: [2023-05-13 09:38:58,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:58,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:58,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:58,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:58,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:58,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +27: [2023-05-13 09:38:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:58,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:58,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +13: [2023-05-13 09:38:58,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:58,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:58,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:58,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:58,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +27: [2023-05-13 09:38:58,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +14: [2023-05-13 09:38:58,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:58,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:58,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. +13: [2023-05-13 09:38:58,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_32-model_01-model_states.pt. + 5: [2023-05-13 09:38:58,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +29: [2023-05-13 09:38:58,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:58,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +29: [2023-05-13 09:38:58,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +15: [2023-05-13 09:38:58,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:58,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:58,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:58,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:58,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:58,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:58,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:58,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +15: [2023-05-13 09:38:58,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:58,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:58,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:58,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:58,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:58,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:58,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:58,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +22: [2023-05-13 09:38:58,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:58,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:58,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:58,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:58,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:58,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:58,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:58,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:58,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:58,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:58,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:58,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:58,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:58,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:58,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:58,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:58,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:58,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:58,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:58,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +29: [2023-05-13 09:38:58,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:58,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:58,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:58,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:58,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:58,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +23: [2023-05-13 09:38:58,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +29: [2023-05-13 09:38:58,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:58,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +23: [2023-05-13 09:38:58,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:58,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:58,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:58,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:58,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:58,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:58,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +22: [2023-05-13 09:38:58,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:58,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +23: [2023-05-13 09:38:58,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +23: [2023-05-13 09:38:58,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:58,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:58,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 3: [2023-05-13 09:38:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +23: [2023-05-13 09:38:58,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:58,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:58,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:58,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 3: [2023-05-13 09:38:58,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:58,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:58,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:58,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +24: [2023-05-13 09:38:58,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:58,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:58,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:58,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 4: [2023-05-13 09:38:58,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:58,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:58,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:58,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:58,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:58,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:58,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:58,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:58,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +24: [2023-05-13 09:38:58,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:58,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:58,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:58,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:58,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:58,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:58,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:58,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:58,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:58,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:58,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:58,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +31: [2023-05-13 09:38:58,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:58,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:58,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:58,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:58,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:58,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +31: [2023-05-13 09:38:58,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +31: [2023-05-13 09:38:58,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +31: [2023-05-13 09:38:58,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +31: [2023-05-13 09:38:58,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +23: [2023-05-13 09:38:58,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:58,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:58,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 8: [2023-05-13 09:38:58,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 6: [2023-05-13 09:38:58,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:58,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +20: [2023-05-13 09:38:58,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:58,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:58,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +20: [2023-05-13 09:38:58,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:58,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:58,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:58,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:58,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:58,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:58,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:58,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:58,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:58,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:58,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:58,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +12: [2023-05-13 09:38:58,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:58,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:58,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:58,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:58,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:58,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:58,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:58,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:58,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:58,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:58,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:58,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:58,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:58,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:58,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:58,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:58,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:58,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:58,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:58,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:58,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:58,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:58,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:58,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:58,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:58,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:58,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:58,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:58,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:58,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:58,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:58,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:58,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:58,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 8: [2023-05-13 09:38:58,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +26: [2023-05-13 09:38:58,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +31: [2023-05-13 09:38:58,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:58,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:58,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:58,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:58,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:58,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:58,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:58,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:58,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:58,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:58,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:58,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:58,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +27: [2023-05-13 09:38:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:58,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:58,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:58,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:58,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:58,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 8: [2023-05-13 09:38:58,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:58,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:58,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:58,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:58,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:58,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:58,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:58,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:58,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:58,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:58,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:58,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:58,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:58,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:58,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:58,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:58,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:58,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:58,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:58,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:58,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:58,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:58,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:58,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:58,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:58,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:58,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:58,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:58,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:58,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:58,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:58,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:58,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:58,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:58,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:58,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:58,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:58,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:58,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:58,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:58,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:58,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:58,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:58,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:58,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:58,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:58,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:58,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:58,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:58,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:58,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:58,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:58,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:58,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:58,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:58,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:58,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:58,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:58,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:58,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:58,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:58,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:58,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:58,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:58,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:58,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:58,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:58,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:58,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:58,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:58,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:58,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:58,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:58,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:58,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:58,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:58,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:58,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:58,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:58,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:58,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:58,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:58,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:58,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:58,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:58,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:58,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +30: [2023-05-13 09:38:58,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:58,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 1: [2023-05-13 09:38:58,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:58,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +18: [2023-05-13 09:38:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:58,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:58,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:58,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +11: [2023-05-13 09:38:58,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:58,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:58,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +16: [2023-05-13 09:38:58,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:58,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +18: [2023-05-13 09:38:58,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:58,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:58,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:58,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:58,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:58,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:58,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +16: [2023-05-13 09:38:58,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:58,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:58,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +18: [2023-05-13 09:38:58,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +20: [2023-05-13 09:38:58,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:58,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:58,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:58,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:58,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:58,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:58,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:58,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:58,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +21: [2023-05-13 09:38:58,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:58,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:58,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:58,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 0: [2023-05-13 09:38:58,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +21: [2023-05-13 09:38:58,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:58,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:58,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:58,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:58,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:58,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:58,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:58,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +15: [2023-05-13 09:38:58,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:58,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:58,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:58,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:58,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +15: [2023-05-13 09:38:58,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:58,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:58,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:58,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:58,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:58,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:58,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:58,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:58,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:58,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:58,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:58,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:58,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:58,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:58,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:58,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +14: [2023-05-13 09:38:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:58,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:58,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:58,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 9: [2023-05-13 09:38:58,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:58,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:58,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +11: [2023-05-13 09:38:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +22: [2023-05-13 09:38:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:58,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:58,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:58,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:58,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:58,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:58,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:58,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:58,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:58,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:58,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:58,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:58,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:58,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:58,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:58,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:58,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:58,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:58,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:58,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:58,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +12: [2023-05-13 09:38:58,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +12: [2023-05-13 09:38:58,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +12: [2023-05-13 09:38:58,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:58,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:58,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:58,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:58,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:58,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:58,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:58,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:58,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:58,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:58,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:58,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:58,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:58,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:58,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:58,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:58,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:58,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:58,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:58,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:58,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:58,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:58,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:58,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:58,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:58,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:58,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:58,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:58,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:59,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:59,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:59,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:59,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:59,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:59,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:59,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:59,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:59,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:59,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:59,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:59,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:59,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:59,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +26: [2023-05-13 09:38:59,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +26: [2023-05-13 09:38:59,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:59,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:59,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:59,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:59,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:59,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:59,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:59,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:59,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +26: [2023-05-13 09:38:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +23: [2023-05-13 09:38:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:59,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:59,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +24: [2023-05-13 09:38:59,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:59,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:59,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:59,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +17: [2023-05-13 09:38:59,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +17: [2023-05-13 09:38:59,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +17: [2023-05-13 09:38:59,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:59,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:59,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:59,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:59,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:59,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:59,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:59,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 6: [2023-05-13 09:38:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:59,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:59,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 6: [2023-05-13 09:38:59,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:59,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:59,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +23: [2023-05-13 09:38:59,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:59,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:59,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:59,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 8: [2023-05-13 09:38:59,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:59,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:59,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:59,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:59,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:59,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:59,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:59,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:59,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:59,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +17: [2023-05-13 09:38:59,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:59,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:59,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:59,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:59,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:59,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:59,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:59,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:59,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:59,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:59,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:59,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 6: [2023-05-13 09:38:59,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +22: [2023-05-13 09:38:59,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:59,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:59,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:59,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:59,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:59,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:59,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:59,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:59,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +26: [2023-05-13 09:38:59,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:59,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:59,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:59,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:59,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:59,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:59,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:59,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:59,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:59,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:59,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:59,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:59,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:59,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:59,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:59,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:59,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:59,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:59,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:59,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:59,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:59,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:59,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:59,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:59,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +13: [2023-05-13 09:38:59,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:59,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:59,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +19: [2023-05-13 09:38:59,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:59,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +13: [2023-05-13 09:38:59,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:59,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:59,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:59,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:59,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:59,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:59,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +21: [2023-05-13 09:38:59,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:59,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:59,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:59,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:59,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +21: [2023-05-13 09:38:59,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +16: [2023-05-13 09:38:59,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +21: [2023-05-13 09:38:59,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:59,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +10: [2023-05-13 09:38:59,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +17: [2023-05-13 09:38:59,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:59,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:59,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:59,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:59,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:59,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:59,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:59,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +24: [2023-05-13 09:38:59,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:59,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:59,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:59,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:59,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:59,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:59,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:59,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:59,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 4: [2023-05-13 09:38:59,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +15: [2023-05-13 09:38:59,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:59,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:59,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:59,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:59,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:59,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +22: [2023-05-13 09:38:59,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:59,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +22: [2023-05-13 09:38:59,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:59,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 4: [2023-05-13 09:38:59,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:59,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:59,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:59,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:59,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:59,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:59,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:59,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +16: [2023-05-13 09:38:59,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:59,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:59,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:59,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:59,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +17: [2023-05-13 09:38:59,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +17: [2023-05-13 09:38:59,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 6: [2023-05-13 09:38:59,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +22: [2023-05-13 09:38:59,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:59,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +17: [2023-05-13 09:38:59,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +22: [2023-05-13 09:38:59,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:59,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +10: [2023-05-13 09:38:59,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +15: [2023-05-13 09:38:59,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:59,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:59,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +26: [2023-05-13 09:38:59,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 1: [2023-05-13 09:38:59,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 1: [2023-05-13 09:38:59,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 1: [2023-05-13 09:38:59,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:59,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +26: [2023-05-13 09:38:59,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:59,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +10: [2023-05-13 09:38:59,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:59,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:59,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +16: [2023-05-13 09:38:59,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:59,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:59,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:59,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:59,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +19: [2023-05-13 09:38:59,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:59,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:59,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:59,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:59,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:59,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 1: [2023-05-13 09:38:59,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:59,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:59,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:59,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:59,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:59,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:59,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:59,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +11: [2023-05-13 09:38:59,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:59,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:59,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:59,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:59,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:59,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:59,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:59,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:59,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:59,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +16: [2023-05-13 09:38:59,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:59,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:59,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +21: [2023-05-13 09:38:59,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:59,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:59,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:59,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:59,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +18: [2023-05-13 09:38:59,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:59,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:59,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:59,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:59,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +16: [2023-05-13 09:38:59,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:59,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:59,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:59,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:59,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:59,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:59,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:59,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 4: [2023-05-13 09:38:59,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:59,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:59,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:59,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:59,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:59,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:59,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:59,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:59,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +30: [2023-05-13 09:38:59,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +30: [2023-05-13 09:38:59,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:59,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +30: [2023-05-13 09:38:59,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:59,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:59,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:59,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:59,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:59,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:59,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +25: [2023-05-13 09:38:59,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:59,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:59,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +12: [2023-05-13 09:38:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:59,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +29: [2023-05-13 09:38:59,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +18: [2023-05-13 09:38:59,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:59,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:59,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:59,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:59,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:59,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:59,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:59,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:59,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:59,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:59,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:59,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:59,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:59,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:59,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:59,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:59,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +18: [2023-05-13 09:38:59,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:59,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:59,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:59,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:59,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:59,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:59,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:59,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:59,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:59,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:59,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:59,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:59,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:59,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:59,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:59,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 8: [2023-05-13 09:38:59,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +25: [2023-05-13 09:38:59,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:59,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:59,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:59,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 8: [2023-05-13 09:38:59,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:59,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:59,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:59,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:59,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:59,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:59,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:59,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:59,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:59,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:59,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:59,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:59,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:59,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:59,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:59,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:59,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:59,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:59,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:59,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:59,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +30: [2023-05-13 09:38:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 3: [2023-05-13 09:38:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 3: [2023-05-13 09:38:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 3: [2023-05-13 09:38:59,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +25: [2023-05-13 09:38:59,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +25: [2023-05-13 09:38:59,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +25: [2023-05-13 09:38:59,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:59,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +25: [2023-05-13 09:38:59,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:59,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:59,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +27: [2023-05-13 09:38:59,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +24: [2023-05-13 09:38:59,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +24: [2023-05-13 09:38:59,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +24: [2023-05-13 09:38:59,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +24: [2023-05-13 09:38:59,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +13: [2023-05-13 09:38:59,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +13: [2023-05-13 09:38:59,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +13: [2023-05-13 09:38:59,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +13: [2023-05-13 09:38:59,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:59,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:59,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +23: [2023-05-13 09:38:59,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:59,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:59,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:59,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +23: [2023-05-13 09:38:59,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +29: [2023-05-13 09:38:59,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:59,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +29: [2023-05-13 09:38:59,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:59,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:59,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:59,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:59,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:59,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:59,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:59,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:59,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:59,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:59,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:59,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:59,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:59,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:59,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +30: [2023-05-13 09:38:59,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:59,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:59,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:59,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:59,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:59,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:59,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:59,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:59,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +28: [2023-05-13 09:38:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +25: [2023-05-13 09:38:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:59,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:59,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:59,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +19: [2023-05-13 09:38:59,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:59,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:59,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:59,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:59,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:59,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:59,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:59,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:59,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:59,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:59,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +20: [2023-05-13 09:38:59,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +23: [2023-05-13 09:38:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +24: [2023-05-13 09:38:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:59,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:59,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:59,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:59,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:59,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:59,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:59,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:59,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:59,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:59,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:59,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:59,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:59,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:59,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:59,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:59,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:59,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:59,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +27: [2023-05-13 09:38:59,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:59,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:59,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +13: [2023-05-13 09:38:59,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +27: [2023-05-13 09:38:59,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:59,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:59,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:59,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:59,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 3: [2023-05-13 09:38:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:59,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:59,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +23: [2023-05-13 09:38:59,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:59,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +27: [2023-05-13 09:38:59,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +20: [2023-05-13 09:38:59,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:59,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:59,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:59,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:59,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:59,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:59,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +27: [2023-05-13 09:38:59,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:59,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +25: [2023-05-13 09:38:59,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:59,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +13: [2023-05-13 09:38:59,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +30: [2023-05-13 09:38:59,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:59,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:59,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:59,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:59,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +23: [2023-05-13 09:38:59,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:59,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:59,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +30: [2023-05-13 09:38:59,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:59,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +28: [2023-05-13 09:38:59,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:59,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:59,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +28: [2023-05-13 09:38:59,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +10: [2023-05-13 09:38:59,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:59,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +23: [2023-05-13 09:38:59,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:59,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 3: [2023-05-13 09:38:59,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:59,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:59,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +25: [2023-05-13 09:38:59,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:59,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:59,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:59,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:59,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:59,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:59,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:59,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 3: [2023-05-13 09:38:59,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:59,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:59,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:59,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 3: [2023-05-13 09:38:59,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +31: [2023-05-13 09:38:59,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +24: [2023-05-13 09:38:59,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 5: [2023-05-13 09:38:59,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:59,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:59,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:59,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:59,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:59,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:59,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:59,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +21: [2023-05-13 09:38:59,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:59,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +19: [2023-05-13 09:38:59,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:59,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:59,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +19: [2023-05-13 09:38:59,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:59,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:59,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:59,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:59,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:59,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +10: [2023-05-13 09:38:59,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:59,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +24: [2023-05-13 09:38:59,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +31: [2023-05-13 09:38:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:59,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:59,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:59,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:59,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:59,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +14: [2023-05-13 09:38:59,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:59,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:59,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 8: [2023-05-13 09:38:59,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:59,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +26: [2023-05-13 09:38:59,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +11: [2023-05-13 09:38:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +11: [2023-05-13 09:38:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +28: [2023-05-13 09:38:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +28: [2023-05-13 09:38:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:59,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:59,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:59,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:59,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:59,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:59,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:59,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +23: [2023-05-13 09:38:59,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:59,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:59,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:59,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:59,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:59,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:59,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... + 5: [2023-05-13 09:38:59,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +20: [2023-05-13 09:38:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +14: [2023-05-13 09:38:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt... +18: [2023-05-13 09:38:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +18: [2023-05-13 09:38:59,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt... +26: [2023-05-13 09:38:59,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:59,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:59,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:59,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:59,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +12: [2023-05-13 09:38:59,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +18: [2023-05-13 09:38:59,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:59,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:59,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:59,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:59,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +14: [2023-05-13 09:38:59,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +26: [2023-05-13 09:38:59,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:59,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 6: [2023-05-13 09:38:59,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 0: [2023-05-13 09:38:59,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:59,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +31: [2023-05-13 09:38:59,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:59,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +26: [2023-05-13 09:38:59,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:59,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +28: [2023-05-13 09:38:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:59,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +11: [2023-05-13 09:38:59,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:59,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:59,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:59,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:59,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:59,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:59,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:59,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:59,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:59,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:59,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:59,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:59,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:59,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:59,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:59,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:59,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:59,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:59,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +12: [2023-05-13 09:38:59,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:59,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +18: [2023-05-13 09:38:59,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:59,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:59,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:59,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:59,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:59,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +20: [2023-05-13 09:38:59,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:59,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:59,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +26: [2023-05-13 09:38:59,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:59,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:59,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 5: [2023-05-13 09:38:59,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +16: [2023-05-13 09:38:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +16: [2023-05-13 09:38:59,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +26: [2023-05-13 09:38:59,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:59,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:59,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:59,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:59,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:38:59,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +20: [2023-05-13 09:38:59,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:59,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:59,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:59,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 7: [2023-05-13 09:38:59,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 7: [2023-05-13 09:38:59,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +17: [2023-05-13 09:38:59,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +15: [2023-05-13 09:38:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +15: [2023-05-13 09:38:59,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:59,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +10: [2023-05-13 09:38:59,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +10: [2023-05-13 09:38:59,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:59,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:59,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:59,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +31: [2023-05-13 09:38:59,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:59,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +18: [2023-05-13 09:38:59,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:59,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 7: [2023-05-13 09:38:59,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:59,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +14: [2023-05-13 09:38:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_00-model_states.pt. +18: [2023-05-13 09:38:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +31: [2023-05-13 09:38:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +31: [2023-05-13 09:38:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +20: [2023-05-13 09:38:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:59,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:59,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:59,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +31: [2023-05-13 09:38:59,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +26: [2023-05-13 09:38:59,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:59,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +18: [2023-05-13 09:38:59,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:59,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +16: [2023-05-13 09:38:59,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:59,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:59,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:59,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:59,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:59,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +18: [2023-05-13 09:38:59,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +31: [2023-05-13 09:38:59,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +23: [2023-05-13 09:38:59,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:59,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +11: [2023-05-13 09:38:59,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +18: [2023-05-13 09:38:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +23: [2023-05-13 09:38:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:38:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:59,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:59,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +11: [2023-05-13 09:38:59,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +10: [2023-05-13 09:38:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +13: [2023-05-13 09:38:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +12: [2023-05-13 09:38:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:59,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:59,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:59,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:59,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:59,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:59,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:59,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:59,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:59,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:59,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:59,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +18: [2023-05-13 09:38:59,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:59,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:59,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:59,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:59,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:59,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:59,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:59,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +17: [2023-05-13 09:38:59,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:59,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:59,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:59,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:59,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +19: [2023-05-13 09:38:59,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +15: [2023-05-13 09:38:59,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:59,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +19: [2023-05-13 09:38:59,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +16: [2023-05-13 09:38:59,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:59,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +17: [2023-05-13 09:38:59,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:59,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:59,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:59,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:59,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:59,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:59,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:59,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:59,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:59,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 3: [2023-05-13 09:38:59,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 3: [2023-05-13 09:38:59,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:59,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +10: [2023-05-13 09:38:59,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:59,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:59,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:59,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:59,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:59,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:59,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +10: [2023-05-13 09:38:59,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +15: [2023-05-13 09:38:59,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:59,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:59,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +21: [2023-05-13 09:38:59,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +21: [2023-05-13 09:38:59,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +21: [2023-05-13 09:38:59,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +13: [2023-05-13 09:38:59,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:59,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +10: [2023-05-13 09:38:59,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +10: [2023-05-13 09:38:59,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +10: [2023-05-13 09:38:59,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... +15: [2023-05-13 09:38:59,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:59,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:59,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:59,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:59,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:59,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:59,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:59,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:59,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:59,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:59,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:59,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:59,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:59,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:59,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +15: [2023-05-13 09:38:59,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. + 2: [2023-05-13 09:38:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +19: [2023-05-13 09:38:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:59,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:59,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:59,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_33-model_01-model_states.pt. +31: [2023-05-13 09:38:59,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:59,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:59,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:59,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:59,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:59,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:59,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 3: [2023-05-13 09:38:59,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 3: [2023-05-13 09:38:59,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:59,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:59,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:59,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:59,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 1: [2023-05-13 09:38:59,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:59,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:59,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +27: [2023-05-13 09:38:59,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:59,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:59,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:59,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +21: [2023-05-13 09:38:59,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:59,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +21: [2023-05-13 09:38:59,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:59,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +21: [2023-05-13 09:38:59,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:59,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +21: [2023-05-13 09:38:59,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +10: [2023-05-13 09:38:59,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:59,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:59,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:59,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:59,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:59,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:59,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:59,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:59,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:38:59,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:59,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +27: [2023-05-13 09:38:59,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:59,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:59,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +22: [2023-05-13 09:38:59,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +22: [2023-05-13 09:38:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +22: [2023-05-13 09:38:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:59,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:59,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:59,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:59,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:59,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +17: [2023-05-13 09:38:59,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:59,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +29: [2023-05-13 09:38:59,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +17: [2023-05-13 09:38:59,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:59,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:59,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:59,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:59,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 4: [2023-05-13 09:38:59,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 1: [2023-05-13 09:38:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 1: [2023-05-13 09:38:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +10: [2023-05-13 09:38:59,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +27: [2023-05-13 09:38:59,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 1: [2023-05-13 09:38:59,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:59,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:59,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +22: [2023-05-13 09:38:59,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +22: [2023-05-13 09:38:59,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt... +29: [2023-05-13 09:38:59,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:59,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:59,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:59,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:59,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:59,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:59,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:59,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:59,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:38:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:59,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:59,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:59,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:59,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:59,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:59,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:59,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +18: [2023-05-13 09:38:59,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:59,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:59,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:59,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:59,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:59,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +22: [2023-05-13 09:38:59,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:59,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:59,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +17: [2023-05-13 09:38:59,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:59,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +28: [2023-05-13 09:38:59,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +28: [2023-05-13 09:38:59,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +28: [2023-05-13 09:38:59,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:59,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +29: [2023-05-13 09:38:59,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +29: [2023-05-13 09:38:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +29: [2023-05-13 09:38:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:59,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:59,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:59,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:59,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:38:59,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +30: [2023-05-13 09:38:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +29: [2023-05-13 09:38:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +29: [2023-05-13 09:38:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_00-model_states.pt. +24: [2023-05-13 09:38:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +22: [2023-05-13 09:38:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:59,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:59,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:59,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:59,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +17: [2023-05-13 09:38:59,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:59,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:59,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:59,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:59,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:59,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +11: [2023-05-13 09:38:59,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:59,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:59,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:59,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +30: [2023-05-13 09:38:59,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:59,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:59,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +30: [2023-05-13 09:38:59,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 8: [2023-05-13 09:38:59,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:59,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +25: [2023-05-13 09:38:59,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:59,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:59,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:59,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:59,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +17: [2023-05-13 09:38:59,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:59,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:59,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +18: [2023-05-13 09:38:59,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:59,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:59,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:59,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:59,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:59,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:59,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:59,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:38:59,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:59,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:59,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 5: [2023-05-13 09:38:59,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +18: [2023-05-13 09:38:59,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:38:59,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:59,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:59,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:59,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:59,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:59,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:59,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:59,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:59,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:59,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:59,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:59,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:59,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:59,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:59,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:59,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:59,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:59,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:59,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:59,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:59,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:59,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:59,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:59,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +28: [2023-05-13 09:38:59,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +28: [2023-05-13 09:38:59,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:59,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:59,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:59,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +22: [2023-05-13 09:38:59,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:59,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:59,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:59,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:59,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:59,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:59,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:59,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:59,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:38:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:59,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +20: [2023-05-13 09:38:59,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:59,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:59,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:59,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:59,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 8: [2023-05-13 09:38:59,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:59,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:59,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:59,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:59,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:59,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:59,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:59,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:38:59,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:59,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:59,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:38:59,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:59,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:59,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:59,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:59,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 8: [2023-05-13 09:38:59,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +30: [2023-05-13 09:38:59,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:59,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:59,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:38:59,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:59,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:59,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:59,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +30: [2023-05-13 09:38:59,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:38:59,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:59,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +31: [2023-05-13 09:38:59,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:59,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:59,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:59,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:59,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +24: [2023-05-13 09:38:59,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:59,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:59,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:59,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:59,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:38:59,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +24: [2023-05-13 09:38:59,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:59,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:59,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:59,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:59,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:59,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:59,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:59,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:59,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:59,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:59,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:59,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:59,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:59,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:38:59,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:59,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:59,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 3: [2023-05-13 09:38:59,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +30: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +25: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +25: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +22: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +31: [2023-05-13 09:38:59,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:59,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +30: [2023-05-13 09:38:59,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:59,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:59,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +26: [2023-05-13 09:38:59,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +26: [2023-05-13 09:38:59,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +26: [2023-05-13 09:38:59,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:59,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +22: [2023-05-13 09:38:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +23: [2023-05-13 09:38:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:59,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:59,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:59,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:59,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +23: [2023-05-13 09:38:59,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +21: [2023-05-13 09:38:59,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:59,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:59,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:59,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:59,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +25: [2023-05-13 09:38:59,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:59,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +30: [2023-05-13 09:38:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:59,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:59,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:59,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +30: [2023-05-13 09:38:59,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +30: [2023-05-13 09:38:59,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 1: [2023-05-13 09:38:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +10: [2023-05-13 09:38:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +10: [2023-05-13 09:38:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +25: [2023-05-13 09:38:59,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:59,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:59,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:59,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +25: [2023-05-13 09:38:59,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:59,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +14: [2023-05-13 09:38:59,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +27: [2023-05-13 09:38:59,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +25: [2023-05-13 09:38:59,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:59,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +27: [2023-05-13 09:38:59,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:59,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +24: [2023-05-13 09:38:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +25: [2023-05-13 09:38:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 7: [2023-05-13 09:38:59,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 7: [2023-05-13 09:38:59,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:59,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:59,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:59,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +24: [2023-05-13 09:38:59,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:38:59,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:59,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:59,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:59,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:38:59,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:59,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:59,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:59,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:59,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:59,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:38:59,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:59,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +21: [2023-05-13 09:38:59,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:59,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:59,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:59,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:59,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:59,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:59,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:59,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:59,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:59,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:59,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:59,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:59,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:59,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:59,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:59,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:59,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:59,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +16: [2023-05-13 09:38:59,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:59,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:59,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:59,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:59,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +31: [2023-05-13 09:38:59,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:59,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:59,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:38:59,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:59,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +25: [2023-05-13 09:38:59,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:38:59,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:59,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +16: [2023-05-13 09:38:59,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +31: [2023-05-13 09:38:59,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 0: [2023-05-13 09:38:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 0: [2023-05-13 09:38:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 7: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +26: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +20: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +14: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +12: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +12: [2023-05-13 09:38:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 6: [2023-05-13 09:38:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:59,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +14: [2023-05-13 09:38:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:59,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 6: [2023-05-13 09:38:59,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:59,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:59,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +16: [2023-05-13 09:38:59,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:59,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 1: [2023-05-13 09:38:59,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:59,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +26: [2023-05-13 09:38:59,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:59,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +14: [2023-05-13 09:38:59,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:59,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +27: [2023-05-13 09:38:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 2: [2023-05-13 09:38:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:59,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:38:59,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +31: [2023-05-13 09:38:59,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:59,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +11: [2023-05-13 09:38:59,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +27: [2023-05-13 09:38:59,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 3: [2023-05-13 09:38:59,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +27: [2023-05-13 09:38:59,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:59,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +18: [2023-05-13 09:38:59,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. +18: [2023-05-13 09:38:59,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 0: [2023-05-13 09:38:59,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:38:59,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_34-model_01-model_states.pt. + 0: [2023-05-13 09:38:59,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:59,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:59,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:59,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:59,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +27: [2023-05-13 09:38:59,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:59,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:59,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:38:59,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:59,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:59,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:59,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:59,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:59,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:38:59,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:59,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +26: [2023-05-13 09:38:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +23: [2023-05-13 09:38:59,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +23: [2023-05-13 09:38:59,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +14: [2023-05-13 09:38:59,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:59,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:59,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:59,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +26: [2023-05-13 09:38:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:59,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +23: [2023-05-13 09:38:59,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:59,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:59,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:38:59,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +16: [2023-05-13 09:38:59,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:59,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:59,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:59,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +19: [2023-05-13 09:38:59,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +19: [2023-05-13 09:38:59,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +23: [2023-05-13 09:38:59,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:59,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +26: [2023-05-13 09:38:59,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:59,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +12: [2023-05-13 09:38:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +12: [2023-05-13 09:38:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +12: [2023-05-13 09:38:59,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:59,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +13: [2023-05-13 09:38:59,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +11: [2023-05-13 09:38:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:59,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +15: [2023-05-13 09:38:59,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +15: [2023-05-13 09:38:59,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:59,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +18: [2023-05-13 09:38:59,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +10: [2023-05-13 09:38:59,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:38:59,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:38:59,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 6: [2023-05-13 09:38:59,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 6: [2023-05-13 09:38:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:59,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:59,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:59,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +10: [2023-05-13 09:38:59,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:59,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:59,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:59,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:38:59,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:59,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:59,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:38:59,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:38:59,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +20: [2023-05-13 09:38:59,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:38:59,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +10: [2023-05-13 09:38:59,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +11: [2023-05-13 09:38:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +19: [2023-05-13 09:38:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:59,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:59,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:59,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:59,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:38:59,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +16: [2023-05-13 09:38:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +19: [2023-05-13 09:38:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:38:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:59,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:38:59,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:38:59,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:59,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:59,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:38:59,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:59,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:59,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:38:59,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:59,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:38:59,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:38:59,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +20: [2023-05-13 09:38:59,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +20: [2023-05-13 09:38:59,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +16: [2023-05-13 09:38:59,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +16: [2023-05-13 09:38:59,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:59,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:59,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:59,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:59,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:59,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +21: [2023-05-13 09:38:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +21: [2023-05-13 09:38:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +21: [2023-05-13 09:38:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +20: [2023-05-13 09:38:59,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +28: [2023-05-13 09:38:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:59,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:38:59,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:59,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:59,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +11: [2023-05-13 09:38:59,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:59,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +13: [2023-05-13 09:38:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:59,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +19: [2023-05-13 09:38:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +18: [2023-05-13 09:38:59,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +13: [2023-05-13 09:38:59,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +28: [2023-05-13 09:38:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:59,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:59,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:59,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:59,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +12: [2023-05-13 09:38:59,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:59,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +18: [2023-05-13 09:38:59,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +13: [2023-05-13 09:38:59,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:59,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:59,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:59,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:59,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:38:59,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:59,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:38:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +12: [2023-05-13 09:38:59,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:59,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:59,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... +19: [2023-05-13 09:38:59,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:59,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:38:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:38:59,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:59,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:38:59,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +19: [2023-05-13 09:38:59,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 6: [2023-05-13 09:38:59,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:38:59,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:59,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:59,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +19: [2023-05-13 09:38:59,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:38:59,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +19: [2023-05-13 09:38:59,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:38:59,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:59,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:59,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:59,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:59,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:59,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:59,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +17: [2023-05-13 09:38:59,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 0: [2023-05-13 09:38:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +15: [2023-05-13 09:38:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:38:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +17: [2023-05-13 09:38:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +15: [2023-05-13 09:38:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:38:59,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt... +17: [2023-05-13 09:38:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:38:59,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:59,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:38:59,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:59,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:59,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:38:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:59,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +21: [2023-05-13 09:38:59,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:59,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +21: [2023-05-13 09:38:59,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +21: [2023-05-13 09:38:59,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +21: [2023-05-13 09:38:59,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:59,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:59,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:59,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:59,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +17: [2023-05-13 09:38:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +27: [2023-05-13 09:38:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:59,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:59,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:59,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:59,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:59,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:59,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:59,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:59,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:59,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:59,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:59,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:59,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:59,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:59,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:59,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:38:59,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. +15: [2023-05-13 09:38:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:59,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:59,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 1: [2023-05-13 09:38:59,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 8: [2023-05-13 09:38:59,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +17: [2023-05-13 09:38:59,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 4: [2023-05-13 09:38:59,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 1: [2023-05-13 09:38:59,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 1: [2023-05-13 09:38:59,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +17: [2023-05-13 09:38:59,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:38:59,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +29: [2023-05-13 09:38:59,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:38:59,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +29: [2023-05-13 09:38:59,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 2: [2023-05-13 09:38:59,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:59,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:59,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:59,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +15: [2023-05-13 09:38:59,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:38:59,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:38:59,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +27: [2023-05-13 09:38:59,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:59,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +17: [2023-05-13 09:38:59,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +29: [2023-05-13 09:38:59,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:59,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:38:59,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +29: [2023-05-13 09:38:59,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:59,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +17: [2023-05-13 09:38:59,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:59,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:59,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +17: [2023-05-13 09:38:59,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +27: [2023-05-13 09:38:59,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:59,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:59,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:59,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:38:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:59,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:59,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:59,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:59,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:59,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +17: [2023-05-13 09:38:59,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +30: [2023-05-13 09:38:59,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:59,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:59,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:59,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:59,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:59,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:59,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:59,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:59,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:38:59,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:59,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:59,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:59,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:59,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:59,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:59,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:59,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:59,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:59,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:59,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:59,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:38:59,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:38:59,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:38:59,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +17: [2023-05-13 09:38:59,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 1: [2023-05-13 09:38:59,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 1: [2023-05-13 09:38:59,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 4: [2023-05-13 09:38:59,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 8: [2023-05-13 09:38:59,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 4: [2023-05-13 09:38:59,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 4: [2023-05-13 09:38:59,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +31: [2023-05-13 09:38:59,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:59,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:59,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:59,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:59,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:59,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:59,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:38:59,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:38:59,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:38:59,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:38:59,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:59,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:59,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:38:59,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:38:59,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:38:59,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:38:59,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:59,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:59,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:59,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:59,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:59,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:59,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:59,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:59,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:59,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:59,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:59,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:59,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:59,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:59,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:59,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:59,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:38:59,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:59,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:59,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:59,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:38:59,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:59,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:59,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:38:59,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:38:59,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:59,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:59,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt... +17: [2023-05-13 09:38:59,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:38:59,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:59,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:59,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:59,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:59,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:59,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:38:59,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 7: [2023-05-13 09:38:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 7: [2023-05-13 09:38:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +25: [2023-05-13 09:38:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +25: [2023-05-13 09:38:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:38:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +28: [2023-05-13 09:38:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +28: [2023-05-13 09:38:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 9: [2023-05-13 09:38:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:59,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:59,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:59,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:59,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:38:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +25: [2023-05-13 09:38:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +25: [2023-05-13 09:38:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +25: [2023-05-13 09:38:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 4: [2023-05-13 09:38:59,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:59,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:59,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:59,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:38:59,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:38:59,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:59,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:38:59,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:38:59,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:59,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:59,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:59,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:59,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:38:59,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:38:59,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 8: [2023-05-13 09:38:59,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:38:59,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:38:59,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:38:59,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:38:59,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:59,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:59,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 7: [2023-05-13 09:38:59,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:38:59,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 8: [2023-05-13 09:38:59,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:59,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:59,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:59,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +31: [2023-05-13 09:38:59,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:59,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:59,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:59,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:59,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:59,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 5: [2023-05-13 09:38:59,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +31: [2023-05-13 09:38:59,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:59,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:38:59,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:38:59,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:59,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:59,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:59,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:59,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:59,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +30: [2023-05-13 09:38:59,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +28: [2023-05-13 09:38:59,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:59,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:59,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:59,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:59,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:59,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +25: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:59,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:38:59,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:38:59,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:59,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:59,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:59,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:59,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 1: [2023-05-13 09:38:59,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:59,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +13: [2023-05-13 09:38:59,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:59,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:38:59,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:59,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:59,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:59,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:38:59,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:59,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:59,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:59,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:38:59,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:59,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:59,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:38:59,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:59,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:59,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:38:59,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:38:59,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:59,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:59,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:38:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:38:59,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:59,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:59,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:59,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:59,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:59,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:59,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:59,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:59,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +16: [2023-05-13 09:38:59,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:59,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:38:59,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:59,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:59,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:59,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:59,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +28: [2023-05-13 09:38:59,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:59,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +26: [2023-05-13 09:38:59,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +26: [2023-05-13 09:38:59,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +28: [2023-05-13 09:38:59,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:59,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +26: [2023-05-13 09:38:59,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:38:59,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:59,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:59,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:59,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:59,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:59,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:38:59,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:59,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:59,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:38:59,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:59,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:59,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +13: [2023-05-13 09:38:59,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:59,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:38:59,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:59,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +30: [2023-05-13 09:38:59,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +26: [2023-05-13 09:38:59,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:59,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 5: [2023-05-13 09:38:59,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:59,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:38:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt... +11: [2023-05-13 09:38:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:59,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:59,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +30: [2023-05-13 09:38:59,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:59,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:38:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:38:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +22: [2023-05-13 09:38:59,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:38:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:38:59,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:38:59,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:38:59,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 5: [2023-05-13 09:38:59,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 5: [2023-05-13 09:38:59,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:59,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:59,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +22: [2023-05-13 09:38:59,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:38:59,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +31: [2023-05-13 09:38:59,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 5: [2023-05-13 09:38:59,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +30: [2023-05-13 09:38:59,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:38:59,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:59,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:38:59,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:38:59,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:59,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:59,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:38:59,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:59,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:59,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +24: [2023-05-13 09:38:59,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:59,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +25: [2023-05-13 09:38:59,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:59,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 6: [2023-05-13 09:38:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... + 6: [2023-05-13 09:38:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt... +25: [2023-05-13 09:38:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt... +26: [2023-05-13 09:38:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:38:59,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:59,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:38:59,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:59,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:59,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:59,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +18: [2023-05-13 09:38:59,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +23: [2023-05-13 09:38:59,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +23: [2023-05-13 09:38:59,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 3: [2023-05-13 09:38:59,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:59,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +31: [2023-05-13 09:38:59,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +30: [2023-05-13 09:38:59,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:59,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +25: [2023-05-13 09:38:59,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +22: [2023-05-13 09:38:59,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:38:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +23: [2023-05-13 09:38:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +22: [2023-05-13 09:38:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:38:59,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:38:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:38:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:38:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +20: [2023-05-13 09:38:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt... +20: [2023-05-13 09:38:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt... +20: [2023-05-13 09:38:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt... +20: [2023-05-13 09:38:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt... +30: [2023-05-13 09:38:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +30: [2023-05-13 09:38:59,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:38:59,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:38:59,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:59,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:59,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:38:59,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:38:59,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 9: [2023-05-13 09:38:59,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:38:59,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:38:59,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +16: [2023-05-13 09:38:59,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:59,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:38:59,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +31: [2023-05-13 09:38:59,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 3: [2023-05-13 09:38:59,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +16: [2023-05-13 09:38:59,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:59,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:38:59,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:59,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:59,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:38:59,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:38:59,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:38:59,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:38:59,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:38:59,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 1: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... +23: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +16: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:59,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:38:59,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:59,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:38:59,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +21: [2023-05-13 09:38:59,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:59,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:59,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:38:59,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:59,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:59,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:59,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:38:59,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:38:59,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:38:59,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:38:59,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:38:59,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:59,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:38:59,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +21: [2023-05-13 09:38:59,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:38:59,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:38:59,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:59,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:38:59,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +24: [2023-05-13 09:38:59,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:38:59,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:38:59,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:38:59,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:38:59,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:59,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:59,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:38:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +14: [2023-05-13 09:38:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +14: [2023-05-13 09:38:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +11: [2023-05-13 09:38:59,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:38:59,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +26: [2023-05-13 09:38:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:38:59,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:59,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:59,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:59,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:38:59,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:38:59,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:59,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +13: [2023-05-13 09:38:59,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +11: [2023-05-13 09:38:59,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +13: [2023-05-13 09:38:59,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +13: [2023-05-13 09:38:59,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +16: [2023-05-13 09:38:59,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:38:59,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:38:59,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:38:59,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +16: [2023-05-13 09:38:59,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:38:59,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:38:59,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:38:59,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:38:59,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +26: [2023-05-13 09:38:59,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:38:59,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:39:00,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +27: [2023-05-13 09:39:00,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:39:00,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt... +22: [2023-05-13 09:39:00,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt... +22: [2023-05-13 09:39:00,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt... +22: [2023-05-13 09:39:00,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt... +27: [2023-05-13 09:39:00,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +10: [2023-05-13 09:39:00,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:39:00,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:39:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +14: [2023-05-13 09:39:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:39:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +10: [2023-05-13 09:39:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:39:00,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:39:00,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:39:00,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:39:00,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +11: [2023-05-13 09:39:00,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +26: [2023-05-13 09:39:00,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:39:00,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +26: [2023-05-13 09:39:00,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:39:00,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:39:00,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:39:00,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +27: [2023-05-13 09:39:00,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:39:00,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +23: [2023-05-13 09:39:00,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +26: [2023-05-13 09:39:00,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:00,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:39:00,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:39:00,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +18: [2023-05-13 09:39:00,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +26: [2023-05-13 09:39:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:39:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:39:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +26: [2023-05-13 09:39:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:39:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:39:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:39:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +26: [2023-05-13 09:39:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:39:00,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:39:00,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:39:00,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:39:00,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:39:00,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:39:00,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:39:00,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:39:00,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:39:00,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:39:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:39:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:39:00,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:39:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:39:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:39:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:39:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +27: [2023-05-13 09:39:00,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +23: [2023-05-13 09:39:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +23: [2023-05-13 09:39:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:39:00,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:39:00,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 2: [2023-05-13 09:39:00,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:39:00,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +18: [2023-05-13 09:39:00,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:39:00,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +20: [2023-05-13 09:39:00,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +20: [2023-05-13 09:39:00,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +20: [2023-05-13 09:39:00,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:39:00,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +23: [2023-05-13 09:39:00,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:39:00,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +18: [2023-05-13 09:39:00,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:39:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:39:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:39:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:39:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:39:00,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:39:00,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:39:00,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:39:00,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:39:00,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:39:00,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:39:00,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt... +21: [2023-05-13 09:39:00,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:39:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:39:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 2: [2023-05-13 09:39:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:39:00,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:39:00,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_35-model_01-model_states.pt. +14: [2023-05-13 09:39:00,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:39:00,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +10: [2023-05-13 09:39:00,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:39:00,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:39:00,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:39:00,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:39:00,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +10: [2023-05-13 09:39:00,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 6: [2023-05-13 09:39:00,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:39:00,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:39:00,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:39:00,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:39:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:39:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:39:00,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:39:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:39:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:39:00,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:39:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:39:00,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:39:00,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:39:00,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:39:00,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:39:00,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:39:00,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +20: [2023-05-13 09:39:00,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:39:00,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:39:00,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:39:00,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:39:00,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:39:00,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:39:00,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:39:00,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:39:00,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:39:00,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:39:00,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:39:00,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:39:00,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:39:00,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:39:00,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:39:00,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:39:00,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:39:00,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +10: [2023-05-13 09:39:00,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:39:00,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +20: [2023-05-13 09:39:00,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:39:00,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +15: [2023-05-13 09:39:00,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:39:00,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:39:00,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:39:00,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +15: [2023-05-13 09:39:00,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +19: [2023-05-13 09:39:00,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +19: [2023-05-13 09:39:00,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:39:00,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:39:00,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:39:00,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:39:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... + 2: [2023-05-13 09:39:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt... +19: [2023-05-13 09:39:00,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +10: [2023-05-13 09:39:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +19: [2023-05-13 09:39:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:39:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:39:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:39:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +24: [2023-05-13 09:39:00,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:39:00,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:39:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:39:00,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +24: [2023-05-13 09:39:00,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:39:00,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +12: [2023-05-13 09:39:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:39:00,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 6: [2023-05-13 09:39:00,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:39:00,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:39:00,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 6: [2023-05-13 09:39:00,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:00,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:39:00,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:39:00,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:39:00,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +28: [2023-05-13 09:39:00,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:39:00,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +28: [2023-05-13 09:39:00,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt... +26: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt... +26: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt... +26: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:39:00,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:39:00,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:39:00,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:39:00,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:39:00,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:39:00,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:39:00,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:39:00,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:00,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:39:00,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 0: > overriding learning rate value to 0.0002 + 0: > overriding minimum learning rate value to 2e-05 + 8: [2023-05-13 09:39:00,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 0: > overriding warmup iterations value to 0 + 0: > overriding total number of iterations value to 1 + 0: > overriding decay style value to cosine +19: [2023-05-13 09:39:00,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:39:00,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:39:00,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:00,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:39:00,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:39:00,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... + 9: [2023-05-13 09:39:00,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... +14: [2023-05-13 09:39:00,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:39:00,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:39:00,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:39:00,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:00,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:00,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:00,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:39:00,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 6: [2023-05-13 09:39:00,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 0: [2023-05-13 09:39:00,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +20: [2023-05-13 09:39:00,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:39:00,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. +14: [2023-05-13 09:39:00,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:39:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt... +24: [2023-05-13 09:39:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt... +24: [2023-05-13 09:39:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt... +24: [2023-05-13 09:39:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt... +20: [2023-05-13 09:39:00,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:39:00,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +14: [2023-05-13 09:39:00,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:39:00,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:39:00,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:39:00,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:39:00,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:39:00,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... + 0: [2023-05-13 09:39:00,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... + 0: [2023-05-13 09:39:00,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... + 0: [2023-05-13 09:39:00,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... + 4: [2023-05-13 09:39:00,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:39:00,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:39:00,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:39:00,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:39:00,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:39:00,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 2: [2023-05-13 09:39:00,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:39:00,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:39:00,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:39:00,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:39:00,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:39:00,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 2: [2023-05-13 09:39:00,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... + 0: [2023-05-13 09:39:00,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:39:00,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:39:00,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:39:00,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:39:00,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:39:00,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt... +12: [2023-05-13 09:39:00,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +12: [2023-05-13 09:39:00,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +12: [2023-05-13 09:39:00,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:39:00,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 6: [2023-05-13 09:39:00,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:39:00,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:39:00,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:39:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:00,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:39:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:39:00,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:39:00,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:39:00,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:00,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:39:00,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:00,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:39:00,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:39:00,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:00,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:39:00,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:39:00,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +19: [2023-05-13 09:39:00,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +12: [2023-05-13 09:39:00,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:39:00,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:39:00,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:39:00,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:39:00,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:39:00,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:39:00,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:39:00,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:39:00,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:39:00,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:39:00,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:39:00,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +29: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:39:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +16: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +12: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:39:00,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:39:00,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +29: [2023-05-13 09:39:00,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +29: [2023-05-13 09:39:00,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:39:00,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:39:00,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:39:00,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +29: [2023-05-13 09:39:00,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +29: [2023-05-13 09:39:00,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:39:00,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:39:00,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:39:00,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:39:00,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:39:00,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:39:00,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:39:00,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:39:00,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:39:00,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:39:00,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +12: [2023-05-13 09:39:00,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:39:00,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:39:00,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:39:00,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:39:00,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:39:00,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:39:00,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:39:00,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:39:00,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:39:00,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:39:00,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:39:00,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:00,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:00,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:00,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:39:00,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:39:00,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:39:00,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:39:00,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:39:00,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:39:00,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:39:00,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:39:00,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:39:00,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:39:00,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 1: [2023-05-13 09:39:00,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:39:00,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:39:00,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:39:00,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:39:00,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:39:00,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:39:00,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:39:00,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:00,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:00,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:39:00,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:39:00,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:39:00,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +14: [2023-05-13 09:39:00,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +14: [2023-05-13 09:39:00,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +14: [2023-05-13 09:39:00,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +28: [2023-05-13 09:39:00,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:00,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:00,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:00,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:00,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:00,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:00,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:00,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:00,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:39:00,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:39:00,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:39:00,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:39:00,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:39:00,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:39:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:39:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:39:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:39:00,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:39:00,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:39:00,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:39:00,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:39:00,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:39:00,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... + 5: [2023-05-13 09:39:00,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... + 5: [2023-05-13 09:39:00,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... + 5: [2023-05-13 09:39:00,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:39:00,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:39:00,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:39:00,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:39:00,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:39:00,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:39:00,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:39:00,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:39:00,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:39:00,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:39:00,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:39:00,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:39:00,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:39:00,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:39:00,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:39:00,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:39:00,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:39:00,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:39:00,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:39:00,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:39:00,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:39:00,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:39:00,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:39:00,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:39:00,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:39:00,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:39:00,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:39:00,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:39:00,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +12: [2023-05-13 09:39:00,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +12: [2023-05-13 09:39:00,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +12: [2023-05-13 09:39:00,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:39:00,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +12: [2023-05-13 09:39:00,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +16: [2023-05-13 09:39:00,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +16: [2023-05-13 09:39:00,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:39:00,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:39:00,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:39:00,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt... +19: [2023-05-13 09:39:00,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt... +19: [2023-05-13 09:39:00,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt... +19: [2023-05-13 09:39:00,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:39:00,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:39:00,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +31: [2023-05-13 09:39:00,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:39:00,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:39:00,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:39:00,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:39:00,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:39:00,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:00,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt... +28: [2023-05-13 09:39:00,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt... +28: [2023-05-13 09:39:00,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt... +28: [2023-05-13 09:39:00,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt... + 8: [2023-05-13 09:39:00,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... + 8: [2023-05-13 09:39:00,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... + 8: [2023-05-13 09:39:00,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... + 8: [2023-05-13 09:39:00,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:39:00,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:39:00,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +17: [2023-05-13 09:39:00,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +17: [2023-05-13 09:39:00,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +17: [2023-05-13 09:39:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 1: [2023-05-13 09:39:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... + 4: [2023-05-13 09:39:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... + 4: [2023-05-13 09:39:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... + 4: [2023-05-13 09:39:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:39:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 1: [2023-05-13 09:39:00,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:39:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:39:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:39:00,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:39:00,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:00,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:39:00,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:39:00,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +15: [2023-05-13 09:39:00,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +23: [2023-05-13 09:39:00,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:39:00,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:39:00,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:39:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:39:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:39:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:39:00,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:39:00,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:39:00,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:39:00,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:39:00,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:39:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:39:00,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:39:00,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:39:00,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:39:00,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:39:00,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:39:00,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:39:00,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:39:00,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:39:00,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +13: [2023-05-13 09:39:00,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:39:00,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +31: [2023-05-13 09:39:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:39:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:39:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt... +31: [2023-05-13 09:39:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt... +23: [2023-05-13 09:39:00,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:39:00,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:39:00,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:39:00,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:39:00,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +31: [2023-05-13 09:39:00,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:39:00,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:39:00,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:39:00,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:39:00,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 4: [2023-05-13 09:39:00,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:39:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:39:00,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 4: [2023-05-13 09:39:00,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 4: [2023-05-13 09:39:00,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +31: [2023-05-13 09:39:00,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:39:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:39:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +21: [2023-05-13 09:39:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +21: [2023-05-13 09:39:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +21: [2023-05-13 09:39:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +31: [2023-05-13 09:39:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:39:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:39:00,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:39:00,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:39:00,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +31: [2023-05-13 09:39:00,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:39:00,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +31: [2023-05-13 09:39:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +22: [2023-05-13 09:39:00,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:39:00,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:39:00,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:39:00,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +17: [2023-05-13 09:39:00,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:39:00,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +17: [2023-05-13 09:39:00,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +17: [2023-05-13 09:39:00,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:39:00,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:39:00,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:39:00,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:39:00,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:39:00,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:39:00,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:39:00,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +21: [2023-05-13 09:39:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +21: [2023-05-13 09:39:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:39:00,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:39:00,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:39:00,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:39:00,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +22: [2023-05-13 09:39:00,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:39:00,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +26: [2023-05-13 09:39:00,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +26: [2023-05-13 09:39:00,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +26: [2023-05-13 09:39:00,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:39:00,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:39:00,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +28: [2023-05-13 09:39:00,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:39:00,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:39:00,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +28: [2023-05-13 09:39:00,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:39:00,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 8: [2023-05-13 09:39:00,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 8: [2023-05-13 09:39:00,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:39:00,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:39:00,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:39:00,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:39:00,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:39:00,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:39:00,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:39:00,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:39:00,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:39:00,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:39:00,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:39:00,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:39:00,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +21: [2023-05-13 09:39:00,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:39:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +28: [2023-05-13 09:39:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:39:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +13: [2023-05-13 09:39:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +13: [2023-05-13 09:39:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:39:00,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:39:00,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +22: [2023-05-13 09:39:00,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:39:00,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:39:00,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +22: [2023-05-13 09:39:00,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:39:00,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +25: [2023-05-13 09:39:00,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +21: [2023-05-13 09:39:00,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +26: [2023-05-13 09:39:00,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:39:00,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +21: [2023-05-13 09:39:00,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +21: [2023-05-13 09:39:00,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +22: [2023-05-13 09:39:00,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:39:00,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +25: [2023-05-13 09:39:00,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +26: [2023-05-13 09:39:00,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +22: [2023-05-13 09:39:00,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:39:00,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:39:00,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +26: [2023-05-13 09:39:00,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:39:00,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:39:00,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:39:00,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 6: [2023-05-13 09:39:00,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:39:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:39:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:39:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +11: [2023-05-13 09:39:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +11: [2023-05-13 09:39:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +24: [2023-05-13 09:39:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:39:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:39:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:39:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:39:00,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:39:00,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:39:00,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:39:00,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:39:00,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:39:00,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:00,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:39:00,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +26: [2023-05-13 09:39:00,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:39:00,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:39:00,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:39:00,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +26: [2023-05-13 09:39:00,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:39:00,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:39:00,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +25: [2023-05-13 09:39:00,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:39:00,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:39:00,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 6: [2023-05-13 09:39:00,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:39:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:39:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +28: [2023-05-13 09:39:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:39:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:39:00,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:39:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +27: [2023-05-13 09:39:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +27: [2023-05-13 09:39:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +27: [2023-05-13 09:39:00,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 8: [2023-05-13 09:39:00,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:39:00,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:39:00,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:39:00,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +28: [2023-05-13 09:39:00,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:39:00,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:39:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:39:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:39:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:39:00,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +24: [2023-05-13 09:39:00,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +30: [2023-05-13 09:39:00,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:39:00,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:39:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:39:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:39:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:39:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:39:00,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:39:00,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:39:00,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:39:00,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +12: [2023-05-13 09:39:00,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +12: [2023-05-13 09:39:00,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +12: [2023-05-13 09:39:00,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:39:00,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +27: [2023-05-13 09:39:00,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +13: [2023-05-13 09:39:00,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:39:00,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +19: [2023-05-13 09:39:00,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +19: [2023-05-13 09:39:00,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +19: [2023-05-13 09:39:00,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:39:00,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +19: [2023-05-13 09:39:00,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:39:00,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +20: [2023-05-13 09:39:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +20: [2023-05-13 09:39:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +20: [2023-05-13 09:39:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +20: [2023-05-13 09:39:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +12: [2023-05-13 09:39:00,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:39:00,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +13: [2023-05-13 09:39:00,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:39:00,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +12: [2023-05-13 09:39:00,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:39:00,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:39:00,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 6: [2023-05-13 09:39:00,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:39:00,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 6: [2023-05-13 09:39:00,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:39:00,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:39:00,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:39:00,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +25: [2023-05-13 09:39:00,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:39:00,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:39:00,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +19: [2023-05-13 09:39:00,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:39:00,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:39:00,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:39:00,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:39:00,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:39:00,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +25: [2023-05-13 09:39:00,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +13: [2023-05-13 09:39:00,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:39:00,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:39:00,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:39:00,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:39:00,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +10: [2023-05-13 09:39:00,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:39:00,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +27: [2023-05-13 09:39:00,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +10: [2023-05-13 09:39:00,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:39:00,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +30: [2023-05-13 09:39:00,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +20: [2023-05-13 09:39:00,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:39:00,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +20: [2023-05-13 09:39:00,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:39:00,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:39:00,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:39:00,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... + 3: [2023-05-13 09:39:00,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... + 3: [2023-05-13 09:39:00,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... + 3: [2023-05-13 09:39:00,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... + 0: [2023-05-13 09:39:00,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:39:00,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +14: [2023-05-13 09:39:00,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +25: [2023-05-13 09:39:00,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:39:00,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:39:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:39:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:39:00,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +30: [2023-05-13 09:39:00,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:39:00,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +25: [2023-05-13 09:39:00,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:39:00,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +29: [2023-05-13 09:39:00,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:39:00,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:39:00,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:39:00,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +24: [2023-05-13 09:39:00,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +27: [2023-05-13 09:39:00,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:39:00,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:39:00,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:39:00,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +30: [2023-05-13 09:39:00,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:39:00,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +29: [2023-05-13 09:39:00,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:39:00,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:39:00,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:39:00,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:39:00,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:39:00,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:39:00,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:39:00,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:39:00,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +10: [2023-05-13 09:39:00,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +24: [2023-05-13 09:39:00,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:39:00,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:39:00,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:39:00,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:39:00,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:39:00,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +11: [2023-05-13 09:39:00,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:39:00,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:39:00,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:39:00,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +14: [2023-05-13 09:39:00,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:39:00,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:39:00,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:39:00,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:39:00,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:39:00,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:39:00,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:39:00,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +11: [2023-05-13 09:39:00,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:39:00,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:39:00,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:39:00,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:39:00,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:39:00,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:39:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:39:00,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:39:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:00,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:39:00,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:39:00,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +15: [2023-05-13 09:39:00,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:39:00,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +20: [2023-05-13 09:39:00,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:39:00,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +18: [2023-05-13 09:39:00,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +18: [2023-05-13 09:39:00,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +29: [2023-05-13 09:39:00,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:39:00,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:39:00,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +15: [2023-05-13 09:39:00,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:39:00,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:00,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:00,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:39:00,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:39:00,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:39:00,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:39:00,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +12: [2023-05-13 09:39:00,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:39:00,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:39:00,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:39:00,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:39:00,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:39:00,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:39:00,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +19: [2023-05-13 09:39:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:39:00,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +19: [2023-05-13 09:39:00,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:39:00,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:39:00,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt. + 0: [2023-05-13 09:39:00,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:39:00,394] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 142 +20: [2023-05-13 09:39:00,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +12: [2023-05-13 09:39:00,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:39:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:39:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:39:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:39:00,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:39:00,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:39:00,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:39:00,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +29: [2023-05-13 09:39:00,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:39:00,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +20: [2023-05-13 09:39:00,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +20: [2023-05-13 09:39:00,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:39:00,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:39:00,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +16: [2023-05-13 09:39:00,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:39:00,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +17: [2023-05-13 09:39:00,401] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 142 +18: [2023-05-13 09:39:00,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:39:00,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +16: [2023-05-13 09:39:00,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:39:00,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +23: [2023-05-13 09:39:00,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:39:00,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:39:00,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:39:00,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:39:00,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:39:00,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:00,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:39:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:39:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +14: [2023-05-13 09:39:00,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +14: [2023-05-13 09:39:00,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:39:00,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:39:00,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +14: [2023-05-13 09:39:00,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +16: [2023-05-13 09:39:00,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:39:00,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:39:00,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:00,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:39:00,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +15: [2023-05-13 09:39:00,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:39:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +11: [2023-05-13 09:39:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +11: [2023-05-13 09:39:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +11: [2023-05-13 09:39:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +23: [2023-05-13 09:39:00,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:39:00,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:39:00,423] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 218 +14: [2023-05-13 09:39:00,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:39:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:39:00,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:39:00,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +15: [2023-05-13 09:39:00,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:39:00,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:39:00,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:39:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:39:00,429] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 216 +29: [2023-05-13 09:39:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:39:00,430] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 218 +29: [2023-05-13 09:39:00,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:39:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:39:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:39:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:39:00,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:39:00,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +29: [2023-05-13 09:39:00,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:39:00,436] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 216 + 3: [2023-05-13 09:39:00,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +23: [2023-05-13 09:39:00,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:39:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 7: [2023-05-13 09:39:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +16: [2023-05-13 09:39:00,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:39:00,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:39:00,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 2: [2023-05-13 09:39:00,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 2: [2023-05-13 09:39:00,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. + 2: [2023-05-13 09:39:00,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_36-model_01-model_states.pt. +23: [2023-05-13 09:39:00,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:39:00,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +16: [2023-05-13 09:39:00,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:39:00,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:39:00,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:39:00,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:39:00,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +29: [2023-05-13 09:39:00,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:39:00,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +23: [2023-05-13 09:39:00,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:39:00,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:39:00,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:39:00,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:39:00,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +23: [2023-05-13 09:39:00,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:39:00,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:39:00,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:39:00,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:39:00,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:39:00,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt. +22: [2023-05-13 09:39:00,479] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 178 + 7: [2023-05-13 09:39:00,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:39:00,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt. +17: [2023-05-13 09:39:00,482] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 140 + 1: [2023-05-13 09:39:00,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 1: [2023-05-13 09:39:00,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 1: [2023-05-13 09:39:00,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 1: [2023-05-13 09:39:00,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +22: [2023-05-13 09:39:00,485] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 178 +26: [2023-05-13 09:39:00,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:39:00,485] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 214 + 7: [2023-05-13 09:39:00,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:39:00,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:39:00,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +17: [2023-05-13 09:39:00,488] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 140 +29: [2023-05-13 09:39:00,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt... +29: [2023-05-13 09:39:00,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt... +29: [2023-05-13 09:39:00,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt... +29: [2023-05-13 09:39:00,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:39:00,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:39:00,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:39:00,491] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 214 + 7: [2023-05-13 09:39:00,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt... +23: [2023-05-13 09:39:00,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:39:00,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:39:00,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt... +23: [2023-05-13 09:39:00,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:39:00,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +16: [2023-05-13 09:39:00,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:39:00,503] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 132 + 4: [2023-05-13 09:39:00,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:39:00,509] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 132 + 7: [2023-05-13 09:39:00,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 4: [2023-05-13 09:39:00,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 4: [2023-05-13 09:39:00,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:39:00,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt. +17: [2023-05-13 09:39:00,522] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 136 + 1: [2023-05-13 09:39:00,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +17: [2023-05-13 09:39:00,528] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 136 + 9: [2023-05-13 09:39:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:39:00,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:39:00,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:39:00,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:39:00,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... + 7: [2023-05-13 09:39:00,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:39:00,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:39:00,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +21: [2023-05-13 09:39:00,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:39:00,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:39:00,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:39:00,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 1: [2023-05-13 09:39:00,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:39:00,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt. +25: [2023-05-13 09:39:00,552] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 200 +26: [2023-05-13 09:39:00,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:39:00,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:39:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +22: [2023-05-13 09:39:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 7: [2023-05-13 09:39:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:39:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +26: [2023-05-13 09:39:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:39:00,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:39:00,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:39:00,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:39:00,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:39:00,559] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 200 +26: [2023-05-13 09:39:00,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +26: [2023-05-13 09:39:00,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:39:00,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt. +30: [2023-05-13 09:39:00,564] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 242 +22: [2023-05-13 09:39:00,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt. +22: [2023-05-13 09:39:00,566] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 182 + 9: [2023-05-13 09:39:00,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. + 9: [2023-05-13 09:39:00,568] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 72 + 9: [2023-05-13 09:39:00,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:39:00,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,569] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 10 +26: [2023-05-13 09:39:00,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:39:00,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:39:00,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:39:00,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:39:00,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +26: [2023-05-13 09:39:00,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:39:00,571] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 242 +11: [2023-05-13 09:39:00,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:39:00,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:39:00,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:39:00,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:39:00,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:39:00,573] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 222 +22: [2023-05-13 09:39:00,574] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 182 + 9: [2023-05-13 09:39:00,574] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 72 +22: [2023-05-13 09:39:00,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:39:00,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:39:00,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,576] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 10 +26: [2023-05-13 09:39:00,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:39:00,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:39:00,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +26: [2023-05-13 09:39:00,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +26: [2023-05-13 09:39:00,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:39:00,579] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 222 +26: [2023-05-13 09:39:00,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +31: [2023-05-13 09:39:00,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:39:00,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:39:00,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:39:00,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:39:00,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt. +20: [2023-05-13 09:39:00,585] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 160 +11: [2023-05-13 09:39:00,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:39:00,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt. +27: [2023-05-13 09:39:00,586] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 220 +24: [2023-05-13 09:39:00,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:39:00,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:39:00,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +24: [2023-05-13 09:39:00,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:39:00,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:39:00,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +17: [2023-05-13 09:39:00,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +17: [2023-05-13 09:39:00,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:39:00,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:39:00,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:39:00,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:39:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:39:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:39:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:39:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +11: [2023-05-13 09:39:00,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:39:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +21: [2023-05-13 09:39:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:39:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +26: [2023-05-13 09:39:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +26: [2023-05-13 09:39:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +20: [2023-05-13 09:39:00,593] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 160 +21: [2023-05-13 09:39:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:39:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +21: [2023-05-13 09:39:00,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:39:00,593] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 220 +21: [2023-05-13 09:39:00,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +31: [2023-05-13 09:39:00,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:39:00,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +21: [2023-05-13 09:39:00,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +22: [2023-05-13 09:39:00,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:39:00,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:39:00,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:39:00,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:39:00,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:39:00,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:39:00,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:39:00,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +27: [2023-05-13 09:39:00,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:39:00,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:39:00,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt. +17: [2023-05-13 09:39:00,600] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 138 + 8: [2023-05-13 09:39:00,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 8: [2023-05-13 09:39:00,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:39:00,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:39:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:39:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:39:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +22: [2023-05-13 09:39:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:39:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:39:00,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:39:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt. +17: [2023-05-13 09:39:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:39:00,607] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 240 +28: [2023-05-13 09:39:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:39:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:39:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +28: [2023-05-13 09:39:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +17: [2023-05-13 09:39:00,608] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 138 +24: [2023-05-13 09:39:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:39:00,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:39:00,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +11: [2023-05-13 09:39:00,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:39:00,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:39:00,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:39:00,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:39:00,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:39:00,612] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 240 +11: [2023-05-13 09:39:00,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +12: [2023-05-13 09:39:00,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:39:00,613] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 100 + 5: [2023-05-13 09:39:00,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:39:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:39:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +27: [2023-05-13 09:39:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:39:00,620] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 100 +31: [2023-05-13 09:39:00,620] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 254 +25: [2023-05-13 09:39:00,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt. +25: [2023-05-13 09:39:00,622] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 204 +11: [2023-05-13 09:39:00,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:39:00,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:39:00,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 8: [2023-05-13 09:39:00,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 8: [2023-05-13 09:39:00,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +11: [2023-05-13 09:39:00,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +20: [2023-05-13 09:39:00,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt. +20: [2023-05-13 09:39:00,626] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 162 + 8: [2023-05-13 09:39:00,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 8: [2023-05-13 09:39:00,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 8: [2023-05-13 09:39:00,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 8: [2023-05-13 09:39:00,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +31: [2023-05-13 09:39:00,626] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 254 +11: [2023-05-13 09:39:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +11: [2023-05-13 09:39:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:39:00,629] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 204 +31: [2023-05-13 09:39:00,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:39:00,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:39:00,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt. +30: [2023-05-13 09:39:00,630] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 246 +11: [2023-05-13 09:39:00,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +11: [2023-05-13 09:39:00,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +20: [2023-05-13 09:39:00,632] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 162 +31: [2023-05-13 09:39:00,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +31: [2023-05-13 09:39:00,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +31: [2023-05-13 09:39:00,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:39:00,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +31: [2023-05-13 09:39:00,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:39:00,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:39:00,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:39:00,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:39:00,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:39:00,637] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 246 +30: [2023-05-13 09:39:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:39:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:39:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:39:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:39:00,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:39:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +31: [2023-05-13 09:39:00,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:39:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +17: [2023-05-13 09:39:00,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:39:00,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:39:00,640] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 210 + 6: [2023-05-13 09:39:00,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:39:00,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,642] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 48 +23: [2023-05-13 09:39:00,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:39:00,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +23: [2023-05-13 09:39:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:39:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +28: [2023-05-13 09:39:00,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:39:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:39:00,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +28: [2023-05-13 09:39:00,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:39:00,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:39:00,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +26: [2023-05-13 09:39:00,647] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 210 + 6: [2023-05-13 09:39:00,648] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 48 +27: [2023-05-13 09:39:00,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:39:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:39:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:39:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +17: [2023-05-13 09:39:00,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +17: [2023-05-13 09:39:00,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:39:00,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +27: [2023-05-13 09:39:00,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +25: [2023-05-13 09:39:00,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:39:00,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 0: [2023-05-13 09:39:00,653] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 6 +20: [2023-05-13 09:39:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:39:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:39:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:39:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:39:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt... +20: [2023-05-13 09:39:00,653] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 166 +25: [2023-05-13 09:39:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:39:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:39:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:39:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:39:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +24: [2023-05-13 09:39:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +24: [2023-05-13 09:39:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:39:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +27: [2023-05-13 09:39:00,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:39:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt. + 5: [2023-05-13 09:39:00,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +24: [2023-05-13 09:39:00,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +22: [2023-05-13 09:39:00,656] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 180 +24: [2023-05-13 09:39:00,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +23: [2023-05-13 09:39:00,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. + 5: [2023-05-13 09:39:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:39:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,658] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 38 +30: [2023-05-13 09:39:00,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,659] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 6 +20: [2023-05-13 09:39:00,659] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 166 +23: [2023-05-13 09:39:00,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:39:00,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:39:00,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:39:00,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +22: [2023-05-13 09:39:00,665] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 180 + 4: [2023-05-13 09:39:00,665] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 38 +20: [2023-05-13 09:39:00,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt. +20: [2023-05-13 09:39:00,667] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 164 +25: [2023-05-13 09:39:00,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:39:00,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:39:00,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +25: [2023-05-13 09:39:00,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt... + 8: [2023-05-13 09:39:00,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt... + 8: [2023-05-13 09:39:00,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:39:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:39:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 8: [2023-05-13 09:39:00,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:39:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:39:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. + 6: [2023-05-13 09:39:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,674] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 54 +20: [2023-05-13 09:39:00,674] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 164 +25: [2023-05-13 09:39:00,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt. +25: [2023-05-13 09:39:00,674] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 206 +12: [2023-05-13 09:39:00,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:39:00,677] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 98 +21: [2023-05-13 09:39:00,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt. +21: [2023-05-13 09:39:00,679] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 170 +25: [2023-05-13 09:39:00,681] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 206 + 6: [2023-05-13 09:39:00,681] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 54 +12: [2023-05-13 09:39:00,684] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 98 +21: [2023-05-13 09:39:00,685] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 170 +24: [2023-05-13 09:39:00,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt. + 6: [2023-05-13 09:39:00,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:39:00,688] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 198 + 6: [2023-05-13 09:39:00,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +14: [2023-05-13 09:39:00,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +14: [2023-05-13 09:39:00,690] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 112 +23: [2023-05-13 09:39:00,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:39:00,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:39:00,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. + 0: [2023-05-13 09:39:00,691] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 2 + 6: [2023-05-13 09:39:00,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:39:00,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:39:00,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +13: [2023-05-13 09:39:00,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:39:00,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:39:00,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:39:00,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 6: [2023-05-13 09:39:00,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 6: [2023-05-13 09:39:00,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:39:00,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:39:00,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 6: [2023-05-13 09:39:00,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 6: [2023-05-13 09:39:00,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:39:00,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:00,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +14: [2023-05-13 09:39:00,696] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 112 + 3: [2023-05-13 09:39:00,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +24: [2023-05-13 09:39:00,697] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 198 +23: [2023-05-13 09:39:00,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:39:00,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,698] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 2 + 3: [2023-05-13 09:39:00,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +20: [2023-05-13 09:39:00,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +20: [2023-05-13 09:39:00,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +20: [2023-05-13 09:39:00,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +20: [2023-05-13 09:39:00,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +15: [2023-05-13 09:39:00,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +15: [2023-05-13 09:39:00,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:39:00,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:39:00,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:39:00,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +30: [2023-05-13 09:39:00,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:39:00,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:39:00,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:39:00,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:39:00,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:39:00,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +30: [2023-05-13 09:39:00,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:39:00,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:39:00,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:39:00,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:39:00,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt. +25: [2023-05-13 09:39:00,708] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 202 +25: [2023-05-13 09:39:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:39:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +25: [2023-05-13 09:39:00,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +20: [2023-05-13 09:39:00,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:39:00,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:39:00,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:00,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:39:00,714] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 202 +15: [2023-05-13 09:39:00,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:39:00,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:39:00,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:39:00,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +20: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 1: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 1: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +20: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +20: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +20: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 1: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +20: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +20: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +20: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +20: [2023-05-13 09:39:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:39:00,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +29: [2023-05-13 09:39:00,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +29: [2023-05-13 09:39:00,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +29: [2023-05-13 09:39:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +14: [2023-05-13 09:39:00,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +14: [2023-05-13 09:39:00,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +14: [2023-05-13 09:39:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:39:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +22: [2023-05-13 09:39:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt. +22: [2023-05-13 09:39:00,723] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 176 +14: [2023-05-13 09:39:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:39:00,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +13: [2023-05-13 09:39:00,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:39:00,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +15: [2023-05-13 09:39:00,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:39:00,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:39:00,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:39:00,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:39:00,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:39:00,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:39:00,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:39:00,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +16: [2023-05-13 09:39:00,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:39:00,728] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 134 + 6: [2023-05-13 09:39:00,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:39:00,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:39:00,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:39:00,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:39:00,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. +15: [2023-05-13 09:39:00,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:39:00,729] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 176 + 0: [2023-05-13 09:39:00,730] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 0 +19: [2023-05-13 09:39:00,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt. +19: [2023-05-13 09:39:00,731] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 158 + 1: [2023-05-13 09:39:00,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 1: [2023-05-13 09:39:00,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:39:00,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +13: [2023-05-13 09:39:00,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:00,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 1: [2023-05-13 09:39:00,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 1: [2023-05-13 09:39:00,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 1: [2023-05-13 09:39:00,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 1: [2023-05-13 09:39:00,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:39:00,735] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 134 + 1: [2023-05-13 09:39:00,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:39:00,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:39:00,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:39:00,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 1: [2023-05-13 09:39:00,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. + 4: [2023-05-13 09:39:00,737] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 34 + 0: [2023-05-13 09:39:00,737] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 0 +14: [2023-05-13 09:39:00,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:39:00,738] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 158 +14: [2023-05-13 09:39:00,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +14: [2023-05-13 09:39:00,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:00,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:39:00,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,739] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 122 +29: [2023-05-13 09:39:00,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +29: [2023-05-13 09:39:00,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +29: [2023-05-13 09:39:00,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +29: [2023-05-13 09:39:00,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +28: [2023-05-13 09:39:00,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:39:00,740] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 228 +29: [2023-05-13 09:39:00,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +15: [2023-05-13 09:39:00,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:39:00,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:00,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +15: [2023-05-13 09:39:00,740] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 120 +14: [2023-05-13 09:39:00,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:39:00,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +29: [2023-05-13 09:39:00,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +29: [2023-05-13 09:39:00,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +29: [2023-05-13 09:39:00,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,743] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 34 + 0: could not find arguments in the checkpoint ... + 0: checkpoint version 3.0 + 3: [2023-05-13 09:39:00,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,746] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 122 +26: [2023-05-13 09:39:00,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:39:00,747] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 208 +28: [2023-05-13 09:39:00,747] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 228 +15: [2023-05-13 09:39:00,747] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 120 + 4: [2023-05-13 09:39:00,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. + 3: [2023-05-13 09:39:00,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,752] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 46 + 3: [2023-05-13 09:39:00,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +20: [2023-05-13 09:39:00,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt... +20: [2023-05-13 09:39:00,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt... +20: [2023-05-13 09:39:00,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt... +20: [2023-05-13 09:39:00,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt... +28: [2023-05-13 09:39:00,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:39:00,754] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 224 + 3: [2023-05-13 09:39:00,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +26: [2023-05-13 09:39:00,755] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 208 +12: [2023-05-13 09:39:00,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:39:00,758] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 102 + 5: [2023-05-13 09:39:00,759] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 46 +28: [2023-05-13 09:39:00,761] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 224 + 4: [2023-05-13 09:39:00,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:00,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. + 5: [2023-05-13 09:39:00,764] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 40 + 4: [2023-05-13 09:39:00,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:00,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 4: [2023-05-13 09:39:00,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 4: [2023-05-13 09:39:00,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:39:00,765] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 102 + 4: [2023-05-13 09:39:00,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 4: [2023-05-13 09:39:00,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 4: [2023-05-13 09:39:00,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 4: [2023-05-13 09:39:00,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 4: [2023-05-13 09:39:00,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 4: [2023-05-13 09:39:00,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:39:00,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt. +19: [2023-05-13 09:39:00,771] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 156 + 5: [2023-05-13 09:39:00,771] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 40 + 4: [2023-05-13 09:39:00,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. + 4: [2023-05-13 09:39:00,772] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 36 +14: [2023-05-13 09:39:00,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +14: [2023-05-13 09:39:00,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +14: [2023-05-13 09:39:00,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +15: [2023-05-13 09:39:00,774] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 124 +14: [2023-05-13 09:39:00,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +19: [2023-05-13 09:39:00,777] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 156 +15: [2023-05-13 09:39:00,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt... +15: [2023-05-13 09:39:00,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt... +15: [2023-05-13 09:39:00,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt... +15: [2023-05-13 09:39:00,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt... +21: [2023-05-13 09:39:00,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:39:00,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:39:00,778] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 230 +21: [2023-05-13 09:39:00,778] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 172 + 4: [2023-05-13 09:39:00,778] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 36 + 1: [2023-05-13 09:39:00,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:39:00,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:39:00,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:39:00,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt... +14: [2023-05-13 09:39:00,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +15: [2023-05-13 09:39:00,782] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 124 + 0: [2023-05-13 09:39:00,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:39:00,784] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 172 +28: [2023-05-13 09:39:00,784] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 230 +14: [2023-05-13 09:39:00,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:39:00,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:39:00,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:39:00,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:39:00,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:39:00,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:39:00,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:39:00,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:39:00,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:39:00,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. + 8: [2023-05-13 09:39:00,788] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 66 +18: [2023-05-13 09:39:00,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:39:00,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:39:00,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:39:00,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +18: [2023-05-13 09:39:00,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:39:00,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +18: [2023-05-13 09:39:00,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +24: [2023-05-13 09:39:00,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:39:00,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt... +29: [2023-05-13 09:39:00,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt... +29: [2023-05-13 09:39:00,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt... +29: [2023-05-13 09:39:00,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:39:00,794] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 196 + 8: [2023-05-13 09:39:00,794] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 66 + 0: [2023-05-13 09:39:00,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. + 0: [2023-05-13 09:39:00,797] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 4 + 0: [2023-05-13 09:39:00,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:39:00,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt. +24: [2023-05-13 09:39:00,800] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 192 + 0: [2023-05-13 09:39:00,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +26: [2023-05-13 09:39:00,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt. +26: [2023-05-13 09:39:00,801] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 212 + 0: [2023-05-13 09:39:00,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 0: [2023-05-13 09:39:00,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 0: [2023-05-13 09:39:00,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 0: [2023-05-13 09:39:00,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 0: [2023-05-13 09:39:00,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 0: [2023-05-13 09:39:00,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +24: [2023-05-13 09:39:00,803] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 196 + 0: [2023-05-13 09:39:00,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,805] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 4 +24: [2023-05-13 09:39:00,805] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 192 + 7: [2023-05-13 09:39:00,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 7: [2023-05-13 09:39:00,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. + 7: [2023-05-13 09:39:00,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,808] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 76 + 7: [2023-05-13 09:39:00,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt... + 4: [2023-05-13 09:39:00,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt... + 4: [2023-05-13 09:39:00,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt... + 4: [2023-05-13 09:39:00,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt... +26: [2023-05-13 09:39:00,809] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 212 +18: [2023-05-13 09:39:00,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,815] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 76 +19: [2023-05-13 09:39:00,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt. +19: [2023-05-13 09:39:00,815] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 154 + 6: [2023-05-13 09:39:00,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. +19: [2023-05-13 09:39:00,822] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 154 + 6: [2023-05-13 09:39:00,822] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 52 + 7: [2023-05-13 09:39:00,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +18: [2023-05-13 09:39:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:39:00,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:39:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +19: [2023-05-13 09:39:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. + 7: [2023-05-13 09:39:00,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 7: [2023-05-13 09:39:00,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,828] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 42 +19: [2023-05-13 09:39:00,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 7: [2023-05-13 09:39:00,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 7: [2023-05-13 09:39:00,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:39:00,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 7: [2023-05-13 09:39:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:39:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:39:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 6: [2023-05-13 09:39:00,829] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 52 + 7: [2023-05-13 09:39:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:39:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:39:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:39:00,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:39:00,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:39:00,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +16: [2023-05-13 09:39:00,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:39:00,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +12: [2023-05-13 09:39:00,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:39:00,832] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 130 +18: [2023-05-13 09:39:00,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +12: [2023-05-13 09:39:00,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +12: [2023-05-13 09:39:00,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,834] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 42 +21: [2023-05-13 09:39:00,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:39:00,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +21: [2023-05-13 09:39:00,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:39:00,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:39:00,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +21: [2023-05-13 09:39:00,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:39:00,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:39:00,838] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 248 +16: [2023-05-13 09:39:00,838] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 130 +22: [2023-05-13 09:39:00,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:39:00,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 0: [2023-05-13 09:39:00,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:39:00,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:39:00,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... + 0: [2023-05-13 09:39:00,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... +22: [2023-05-13 09:39:00,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +22: [2023-05-13 09:39:00,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. + 9: [2023-05-13 09:39:00,841] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 78 +13: [2023-05-13 09:39:00,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +13: [2023-05-13 09:39:00,843] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 106 +19: [2023-05-13 09:39:00,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:39:00,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt. +24: [2023-05-13 09:39:00,844] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 194 +19: [2023-05-13 09:39:00,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:39:00,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +19: [2023-05-13 09:39:00,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:39:00,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +31: [2023-05-13 09:39:00,846] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 248 + 9: [2023-05-13 09:39:00,847] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 78 +16: [2023-05-13 09:39:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:39:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:39:00,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:39:00,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:39:00,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:39:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +19: [2023-05-13 09:39:00,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +19: [2023-05-13 09:39:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:39:00,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:39:00,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:39:00,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +19: [2023-05-13 09:39:00,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:39:00,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +12: [2023-05-13 09:39:00,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:39:00,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:39:00,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:39:00,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:39:00,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:39:00,850] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 106 +12: [2023-05-13 09:39:00,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +16: [2023-05-13 09:39:00,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:39:00,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:39:00,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +12: [2023-05-13 09:39:00,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +16: [2023-05-13 09:39:00,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +24: [2023-05-13 09:39:00,851] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 194 +21: [2023-05-13 09:39:00,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:39:00,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:39:00,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:39:00,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:39:00,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:39:00,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:39:00,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:39:00,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:39:00,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +12: [2023-05-13 09:39:00,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +12: [2023-05-13 09:39:00,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +12: [2023-05-13 09:39:00,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:39:00,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:39:00,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +21: [2023-05-13 09:39:00,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:39:00,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:39:00,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:39:00,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +12: [2023-05-13 09:39:00,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +12: [2023-05-13 09:39:00,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +21: [2023-05-13 09:39:00,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +21: [2023-05-13 09:39:00,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +21: [2023-05-13 09:39:00,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +21: [2023-05-13 09:39:00,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +12: [2023-05-13 09:39:00,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +12: [2023-05-13 09:39:00,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +12: [2023-05-13 09:39:00,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +12: [2023-05-13 09:39:00,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:39:00,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:39:00,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:39:00,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:39:00,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:39:00,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:39:00,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +22: [2023-05-13 09:39:00,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:39:00,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:39:00,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +22: [2023-05-13 09:39:00,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:39:00,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:39:00,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +22: [2023-05-13 09:39:00,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:39:00,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:39:00,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +22: [2023-05-13 09:39:00,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:39:00,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +13: [2023-05-13 09:39:00,870] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 108 +28: [2023-05-13 09:39:00,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt. +28: [2023-05-13 09:39:00,872] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 226 +13: [2023-05-13 09:39:00,876] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 108 +28: [2023-05-13 09:39:00,880] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 226 + 7: [2023-05-13 09:39:00,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt... + 7: [2023-05-13 09:39:00,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt... + 7: [2023-05-13 09:39:00,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt... + 7: [2023-05-13 09:39:00,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt... +11: [2023-05-13 09:39:00,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +11: [2023-05-13 09:39:00,883] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 88 +19: [2023-05-13 09:39:00,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt. +19: [2023-05-13 09:39:00,885] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 152 +21: [2023-05-13 09:39:00,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt... +21: [2023-05-13 09:39:00,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt... +21: [2023-05-13 09:39:00,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:39:00,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:39:00,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:39:00,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt... +16: [2023-05-13 09:39:00,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt... +21: [2023-05-13 09:39:00,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt... +11: [2023-05-13 09:39:00,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:39:00,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:39:00,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:39:00,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:39:00,889] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 88 +19: [2023-05-13 09:39:00,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt... +19: [2023-05-13 09:39:00,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt... +19: [2023-05-13 09:39:00,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt... +19: [2023-05-13 09:39:00,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt... +19: [2023-05-13 09:39:00,892] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 152 +12: [2023-05-13 09:39:00,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +12: [2023-05-13 09:39:00,897] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 96 +22: [2023-05-13 09:39:00,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt... +22: [2023-05-13 09:39:00,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt... +22: [2023-05-13 09:39:00,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt... +22: [2023-05-13 09:39:00,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt... +12: [2023-05-13 09:39:00,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt... +12: [2023-05-13 09:39:00,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt... +12: [2023-05-13 09:39:00,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt... +12: [2023-05-13 09:39:00,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:39:00,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +13: [2023-05-13 09:39:00,900] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 104 +11: [2023-05-13 09:39:00,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +11: [2023-05-13 09:39:00,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +12: [2023-05-13 09:39:00,904] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 96 +11: [2023-05-13 09:39:00,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:39:00,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:39:00,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. + 9: [2023-05-13 09:39:00,905] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 74 +11: [2023-05-13 09:39:00,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. + 3: [2023-05-13 09:39:00,906] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 26 +13: [2023-05-13 09:39:00,906] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 104 +11: [2023-05-13 09:39:00,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:00,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +14: [2023-05-13 09:39:00,906] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 114 + 8: [2023-05-13 09:39:00,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. +11: [2023-05-13 09:39:00,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,907] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 70 +11: [2023-05-13 09:39:00,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +11: [2023-05-13 09:39:00,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +11: [2023-05-13 09:39:00,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +11: [2023-05-13 09:39:00,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +11: [2023-05-13 09:39:00,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +31: [2023-05-13 09:39:00,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:39:00,910] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 252 + 8: [2023-05-13 09:39:00,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. + 8: [2023-05-13 09:39:00,911] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 68 + 9: [2023-05-13 09:39:00,911] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 74 + 3: [2023-05-13 09:39:00,913] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 26 + 8: [2023-05-13 09:39:00,915] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 70 +14: [2023-05-13 09:39:00,915] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 114 +31: [2023-05-13 09:39:00,916] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 252 +11: [2023-05-13 09:39:00,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +11: [2023-05-13 09:39:00,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +11: [2023-05-13 09:39:00,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +11: [2023-05-13 09:39:00,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 8: [2023-05-13 09:39:00,918] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 68 +14: [2023-05-13 09:39:00,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +14: [2023-05-13 09:39:00,919] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 116 +30: [2023-05-13 09:39:00,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt. +30: [2023-05-13 09:39:00,925] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 244 +14: [2023-05-13 09:39:00,926] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 116 +25: [2023-05-13 09:39:00,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +25: [2023-05-13 09:39:00,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +25: [2023-05-13 09:39:00,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +25: [2023-05-13 09:39:00,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:39:00,932] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 244 + 5: [2023-05-13 09:39:00,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 8: [2023-05-13 09:39:00,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. + 8: [2023-05-13 09:39:00,939] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 64 +30: [2023-05-13 09:39:00,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:39:00,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:39:00,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +30: [2023-05-13 09:39:00,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:39:00,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +18: [2023-05-13 09:39:00,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +18: [2023-05-13 09:39:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +18: [2023-05-13 09:39:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +25: [2023-05-13 09:39:00,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 8: [2023-05-13 09:39:00,946] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 64 +25: [2023-05-13 09:39:00,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:39:00,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +25: [2023-05-13 09:39:00,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:39:00,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:39:00,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +25: [2023-05-13 09:39:00,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:39:00,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +25: [2023-05-13 09:39:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:39:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +25: [2023-05-13 09:39:00,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +25: [2023-05-13 09:39:00,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +25: [2023-05-13 09:39:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +25: [2023-05-13 09:39:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +25: [2023-05-13 09:39:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +25: [2023-05-13 09:39:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +15: [2023-05-13 09:39:00,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +15: [2023-05-13 09:39:00,954] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 126 +13: [2023-05-13 09:39:00,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +13: [2023-05-13 09:39:00,954] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 110 + 5: [2023-05-13 09:39:00,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:39:00,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:39:00,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 5: [2023-05-13 09:39:00,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 5: [2023-05-13 09:39:00,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:39:00,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:39:00,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:39:00,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:39:00,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:39:00,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:39:00,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:39:00,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:39:00,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:39:00,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +30: [2023-05-13 09:39:00,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:39:00,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 5: [2023-05-13 09:39:00,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:39:00,961] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 110 +15: [2023-05-13 09:39:00,961] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 126 +30: [2023-05-13 09:39:00,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:39:00,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:39:00,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:39:00,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +30: [2023-05-13 09:39:00,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +30: [2023-05-13 09:39:00,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +30: [2023-05-13 09:39:00,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. + 3: [2023-05-13 09:39:00,969] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 28 + 4: [2023-05-13 09:39:00,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. +11: [2023-05-13 09:39:00,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt... +11: [2023-05-13 09:39:00,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt... + 4: [2023-05-13 09:39:00,971] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 32 +11: [2023-05-13 09:39:00,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt... +11: [2023-05-13 09:39:00,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:39:00,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:39:00,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:39:00,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt. +31: [2023-05-13 09:39:00,973] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 250 +23: [2023-05-13 09:39:00,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:39:00,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,975] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 28 +31: [2023-05-13 09:39:00,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:39:00,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:39:00,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:39:00,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 4: [2023-05-13 09:39:00,978] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 32 +31: [2023-05-13 09:39:00,980] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 250 +25: [2023-05-13 09:39:00,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt... +25: [2023-05-13 09:39:00,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt... +25: [2023-05-13 09:39:00,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt... +25: [2023-05-13 09:39:00,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:39:00,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:39:00,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +23: [2023-05-13 09:39:00,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +23: [2023-05-13 09:39:00,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:39:00,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:00,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +14: [2023-05-13 09:39:00,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +14: [2023-05-13 09:39:00,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +14: [2023-05-13 09:39:00,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +23: [2023-05-13 09:39:00,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:39:00,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:39:00,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +23: [2023-05-13 09:39:00,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +28: [2023-05-13 09:39:00,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:39:00,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:39:00,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +31: [2023-05-13 09:39:00,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +31: [2023-05-13 09:39:00,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:39:00,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +28: [2023-05-13 09:39:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +28: [2023-05-13 09:39:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:39:00,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 3: [2023-05-13 09:39:00,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +31: [2023-05-13 09:39:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +31: [2023-05-13 09:39:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +31: [2023-05-13 09:39:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +31: [2023-05-13 09:39:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +31: [2023-05-13 09:39:00,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 3: [2023-05-13 09:39:00,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:39:00,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:39:00,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +31: [2023-05-13 09:39:00,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +11: [2023-05-13 09:39:00,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. + 9: [2023-05-13 09:39:00,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 9: [2023-05-13 09:39:00,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +11: [2023-05-13 09:39:00,995] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 94 + 9: [2023-05-13 09:39:00,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 5: [2023-05-13 09:39:00,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:39:00,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:39:00,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt... + 5: [2023-05-13 09:39:00,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:39:00,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:39:00,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +13: [2023-05-13 09:39:00,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +13: [2023-05-13 09:39:00,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +13: [2023-05-13 09:39:00,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 3: [2023-05-13 09:39:00,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:39:01,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:39:01,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:39:01,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +23: [2023-05-13 09:39:01,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +30: [2023-05-13 09:39:01,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt... +30: [2023-05-13 09:39:01,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt... +30: [2023-05-13 09:39:01,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt... +11: [2023-05-13 09:39:01,003] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 94 +30: [2023-05-13 09:39:01,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:39:01,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. + 3: [2023-05-13 09:39:01,005] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 24 +14: [2023-05-13 09:39:01,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:39:01,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:01,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:39:01,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:39:01,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:01,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +14: [2023-05-13 09:39:01,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +14: [2023-05-13 09:39:01,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:01,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +14: [2023-05-13 09:39:01,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:39:01,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:01,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +11: [2023-05-13 09:39:01,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +11: [2023-05-13 09:39:01,010] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 90 + 9: [2023-05-13 09:39:01,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:01,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:01,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:01,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:01,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:39:01,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +14: [2023-05-13 09:39:01,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +14: [2023-05-13 09:39:01,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 9: [2023-05-13 09:39:01,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +14: [2023-05-13 09:39:01,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +28: [2023-05-13 09:39:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +28: [2023-05-13 09:39:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +28: [2023-05-13 09:39:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +28: [2023-05-13 09:39:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +28: [2023-05-13 09:39:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +14: [2023-05-13 09:39:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +14: [2023-05-13 09:39:01,011] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 118 + 9: [2023-05-13 09:39:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 3: [2023-05-13 09:39:01,012] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 24 + 9: [2023-05-13 09:39:01,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:39:01,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:39:01,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +28: [2023-05-13 09:39:01,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +28: [2023-05-13 09:39:01,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +28: [2023-05-13 09:39:01,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +28: [2023-05-13 09:39:01,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 9: [2023-05-13 09:39:01,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:39:01,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 9: [2023-05-13 09:39:01,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 9: [2023-05-13 09:39:01,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:39:01,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 9: [2023-05-13 09:39:01,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:39:01,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 9: [2023-05-13 09:39:01,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:39:01,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 9: [2023-05-13 09:39:01,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +11: [2023-05-13 09:39:01,017] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 90 +13: [2023-05-13 09:39:01,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +14: [2023-05-13 09:39:01,018] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 118 +13: [2023-05-13 09:39:01,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:39:01,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:39:01,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:39:01,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +13: [2023-05-13 09:39:01,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:39:01,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:39:01,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:39:01,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +13: [2023-05-13 09:39:01,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:39:01,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:39:01,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:39:01,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:39:01,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:39:01,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +13: [2023-05-13 09:39:01,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:39:01,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. + 7: [2023-05-13 09:39:01,023] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 62 + 7: [2023-05-13 09:39:01,029] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 62 +31: [2023-05-13 09:39:01,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:39:01,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:39:01,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt... +31: [2023-05-13 09:39:01,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:39:01,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:39:01,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:39:01,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +17: [2023-05-13 09:39:01,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 3: [2023-05-13 09:39:01,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:39:01,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:39:01,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt... + 3: [2023-05-13 09:39:01,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt... + 2: [2023-05-13 09:39:01,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +27: [2023-05-13 09:39:01,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +27: [2023-05-13 09:39:01,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +27: [2023-05-13 09:39:01,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +27: [2023-05-13 09:39:01,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 2: [2023-05-13 09:39:01,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:39:01,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +24: [2023-05-13 09:39:01,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:39:01,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:39:01,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +24: [2023-05-13 09:39:01,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 2: [2023-05-13 09:39:01,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:39:01,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... + 2: [2023-05-13 09:39:01,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +14: [2023-05-13 09:39:01,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt... +14: [2023-05-13 09:39:01,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt... +14: [2023-05-13 09:39:01,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt... +14: [2023-05-13 09:39:01,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:39:01,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:39:01,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:39:01,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 9: [2023-05-13 09:39:01,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt... + 9: [2023-05-13 09:39:01,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt... + 9: [2023-05-13 09:39:01,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt... + 9: [2023-05-13 09:39:01,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt... +29: [2023-05-13 09:39:01,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:39:01,053] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 236 +17: [2023-05-13 09:39:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:39:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +17: [2023-05-13 09:39:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +28: [2023-05-13 09:39:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt... +28: [2023-05-13 09:39:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt... +28: [2023-05-13 09:39:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt... +28: [2023-05-13 09:39:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:39:01,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:39:01,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:39:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +17: [2023-05-13 09:39:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +17: [2023-05-13 09:39:01,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +17: [2023-05-13 09:39:01,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +17: [2023-05-13 09:39:01,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +17: [2023-05-13 09:39:01,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +17: [2023-05-13 09:39:01,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +17: [2023-05-13 09:39:01,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:39:01,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:39:01,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:39:01,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +27: [2023-05-13 09:39:01,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:39:01,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:39:01,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:39:01,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +27: [2023-05-13 09:39:01,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +27: [2023-05-13 09:39:01,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +13: [2023-05-13 09:39:01,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:39:01,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:39:01,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt... +13: [2023-05-13 09:39:01,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:39:01,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:39:01,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +29: [2023-05-13 09:39:01,059] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 236 +27: [2023-05-13 09:39:01,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:39:01,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:39:01,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +27: [2023-05-13 09:39:01,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:39:01,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +27: [2023-05-13 09:39:01,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +27: [2023-05-13 09:39:01,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +27: [2023-05-13 09:39:01,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +24: [2023-05-13 09:39:01,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:39:01,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:39:01,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +24: [2023-05-13 09:39:01,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:39:01,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +24: [2023-05-13 09:39:01,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:39:01,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +24: [2023-05-13 09:39:01,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +24: [2023-05-13 09:39:01,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +24: [2023-05-13 09:39:01,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +24: [2023-05-13 09:39:01,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +24: [2023-05-13 09:39:01,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +24: [2023-05-13 09:39:01,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 2: [2023-05-13 09:39:01,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +23: [2023-05-13 09:39:01,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:39:01,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:39:01,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:39:01,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt... + 2: [2023-05-13 09:39:01,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 7: [2023-05-13 09:39:01,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. + 7: [2023-05-13 09:39:01,087] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 56 + 1: [2023-05-13 09:39:01,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. + 1: [2023-05-13 09:39:01,088] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 14 +17: [2023-05-13 09:39:01,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:39:01,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:39:01,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt... +17: [2023-05-13 09:39:01,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:39:01,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:39:01,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:39:01,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt... +27: [2023-05-13 09:39:01,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt... + 1: [2023-05-13 09:39:01,095] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 14 +26: [2023-05-13 09:39:01,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt. +26: [2023-05-13 09:39:01,095] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 209 + 7: [2023-05-13 09:39:01,095] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 56 + 2: [2023-05-13 09:39:01,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:39:01,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +23: [2023-05-13 09:39:01,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt. +23: [2023-05-13 09:39:01,100] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 186 +18: [2023-05-13 09:39:01,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:39:01,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +26: [2023-05-13 09:39:01,101] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 209 + 2: [2023-05-13 09:39:01,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +24: [2023-05-13 09:39:01,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:39:01,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:39:01,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt... +24: [2023-05-13 09:39:01,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt... +23: [2023-05-13 09:39:01,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt. +23: [2023-05-13 09:39:01,106] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 190 +23: [2023-05-13 09:39:01,108] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 186 +23: [2023-05-13 09:39:01,112] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 190 +11: [2023-05-13 09:39:01,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +11: [2023-05-13 09:39:01,119] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 92 +11: [2023-05-13 09:39:01,126] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 92 + 3: [2023-05-13 09:39:01,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. + 3: [2023-05-13 09:39:01,134] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 30 + 3: [2023-05-13 09:39:01,141] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 30 +18: [2023-05-13 09:39:01,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:39:01,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +18: [2023-05-13 09:39:01,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:39:01,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:39:01,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:39:01,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +18: [2023-05-13 09:39:01,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:01,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +29: [2023-05-13 09:39:01,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:39:01,152] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 232 +26: [2023-05-13 09:39:01,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt. +26: [2023-05-13 09:39:01,156] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 215 +29: [2023-05-13 09:39:01,158] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 232 + 1: [2023-05-13 09:39:01,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. + 1: [2023-05-13 09:39:01,161] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 12 +26: [2023-05-13 09:39:01,163] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 215 + 1: [2023-05-13 09:39:01,167] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 12 +23: [2023-05-13 09:39:01,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt. +23: [2023-05-13 09:39:01,172] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 184 +23: [2023-05-13 09:39:01,180] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 184 +18: [2023-05-13 09:39:01,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:01,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:01,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:01,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:01,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:01,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:01,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:01,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:01,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:01,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:01,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:01,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:01,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:01,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:01,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:39:01,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt. +21: [2023-05-13 09:39:01,193] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 168 +21: [2023-05-13 09:39:01,199] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 168 +29: [2023-05-13 09:39:01,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:39:01,200] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 238 + 2: [2023-05-13 09:39:01,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:39:01,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:39:01,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +29: [2023-05-13 09:39:01,206] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 238 +29: [2023-05-13 09:39:01,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt. +29: [2023-05-13 09:39:01,213] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 234 + 2: [2023-05-13 09:39:01,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. + 2: [2023-05-13 09:39:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +29: [2023-05-13 09:39:01,219] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 234 + 2: [2023-05-13 09:39:01,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:39:01,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt. + 4: [2023-05-13 09:39:01,229] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 39 + 2: [2023-05-13 09:39:01,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt... +18: [2023-05-13 09:39:01,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt... +18: [2023-05-13 09:39:01,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt... +18: [2023-05-13 09:39:01,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:39:01,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 4: [2023-05-13 09:39:01,236] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 39 + 7: [2023-05-13 09:39:01,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. + 7: [2023-05-13 09:39:01,239] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 60 + 2: [2023-05-13 09:39:01,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 7: [2023-05-13 09:39:01,249] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 60 + 2: [2023-05-13 09:39:01,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +21: [2023-05-13 09:39:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:39:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +21: [2023-05-13 09:39:01,259] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 174 + 2: [2023-05-13 09:39:01,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 2: [2023-05-13 09:39:01,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 6: [2023-05-13 09:39:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:39:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:39:01,266] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 174 + 2: [2023-05-13 09:39:01,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 6: [2023-05-13 09:39:01,266] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 53 + 2: [2023-05-13 09:39:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 6: [2023-05-13 09:39:01,272] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 53 + 2: [2023-05-13 09:39:01,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... + 2: [2023-05-13 09:39:01,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:39:01,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt. +20: [2023-05-13 09:39:01,294] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 165 + 8: [2023-05-13 09:39:01,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt. + 8: [2023-05-13 09:39:01,295] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 67 + 8: [2023-05-13 09:39:01,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt. + 8: [2023-05-13 09:39:01,296] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 69 + 2: [2023-05-13 09:39:01,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +20: [2023-05-13 09:39:01,300] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 165 + 8: [2023-05-13 09:39:01,301] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 67 +26: [2023-05-13 09:39:01,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt. +26: [2023-05-13 09:39:01,302] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 213 + 8: [2023-05-13 09:39:01,303] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 69 + 8: [2023-05-13 09:39:01,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt. + 8: [2023-05-13 09:39:01,304] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 71 +22: [2023-05-13 09:39:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt. +22: [2023-05-13 09:39:01,306] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 179 + 1: [2023-05-13 09:39:01,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:39:01,308] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 9 +26: [2023-05-13 09:39:01,309] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 213 +16: [2023-05-13 09:39:01,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt. +16: [2023-05-13 09:39:01,310] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 129 +22: [2023-05-13 09:39:01,312] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 179 + 8: [2023-05-13 09:39:01,312] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 71 +12: [2023-05-13 09:39:01,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt. +12: [2023-05-13 09:39:01,314] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 103 +15: [2023-05-13 09:39:01,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt. +15: [2023-05-13 09:39:01,316] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 123 +16: [2023-05-13 09:39:01,316] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 129 + 2: [2023-05-13 09:39:01,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:39:01,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:39:01,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:39:01,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... + 1: [2023-05-13 09:39:01,317] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 9 +16: [2023-05-13 09:39:01,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt. +16: [2023-05-13 09:39:01,321] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 133 +29: [2023-05-13 09:39:01,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt. +29: [2023-05-13 09:39:01,321] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 235 +12: [2023-05-13 09:39:01,321] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 103 +15: [2023-05-13 09:39:01,322] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 123 +18: [2023-05-13 09:39:01,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:39:01,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:39:01,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:39:01,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt. +16: [2023-05-13 09:39:01,324] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 128 +18: [2023-05-13 09:39:01,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +16: [2023-05-13 09:39:01,327] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 133 +29: [2023-05-13 09:39:01,327] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 235 +16: [2023-05-13 09:39:01,331] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 128 +18: [2023-05-13 09:39:01,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:01,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:01,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:01,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:01,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:01,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:01,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +18: [2023-05-13 09:39:01,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:01,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:39:01,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:39:01,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +18: [2023-05-13 09:39:01,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +16: [2023-05-13 09:39:01,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt. +16: [2023-05-13 09:39:01,345] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 131 +16: [2023-05-13 09:39:01,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:39:01,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +18: [2023-05-13 09:39:01,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +18: [2023-05-13 09:39:01,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:39:01,346] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 135 +18: [2023-05-13 09:39:01,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +16: [2023-05-13 09:39:01,352] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 131 +16: [2023-05-13 09:39:01,352] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 135 + 4: [2023-05-13 09:39:01,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt. + 4: [2023-05-13 09:39:01,355] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 35 + 4: [2023-05-13 09:39:01,362] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 35 +21: [2023-05-13 09:39:01,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:39:01,366] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 171 +21: [2023-05-13 09:39:01,373] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 171 + 6: [2023-05-13 09:39:01,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt. + 6: [2023-05-13 09:39:01,383] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 51 + 6: [2023-05-13 09:39:01,390] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 51 + 4: [2023-05-13 09:39:01,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt. + 4: [2023-05-13 09:39:01,391] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 37 + 0: [2023-05-13 09:39:01,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. + 0: [2023-05-13 09:39:01,392] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 7 + 6: [2023-05-13 09:39:01,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt. + 6: [2023-05-13 09:39:01,393] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 55 +26: [2023-05-13 09:39:01,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt. +26: [2023-05-13 09:39:01,393] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 211 + 1: [2023-05-13 09:39:01,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:39:01,396] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 11 + 4: [2023-05-13 09:39:01,397] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 37 + 0: [2023-05-13 09:39:01,398] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 7 + 6: [2023-05-13 09:39:01,399] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 55 +26: [2023-05-13 09:39:01,399] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 211 +18: [2023-05-13 09:39:01,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt... +18: [2023-05-13 09:39:01,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt... +18: [2023-05-13 09:39:01,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt... +18: [2023-05-13 09:39:01,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt... + 6: [2023-05-13 09:39:01,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:39:01,402] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 11 + 6: [2023-05-13 09:39:01,402] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 49 +28: [2023-05-13 09:39:01,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt. +28: [2023-05-13 09:39:01,403] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 227 + 5: [2023-05-13 09:39:01,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:39:01,404] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 47 +28: [2023-05-13 09:39:01,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt. +28: [2023-05-13 09:39:01,406] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 225 +29: [2023-05-13 09:39:01,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt. +29: [2023-05-13 09:39:01,406] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 237 + 6: [2023-05-13 09:39:01,408] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 49 +23: [2023-05-13 09:39:01,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt. +23: [2023-05-13 09:39:01,409] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 188 +28: [2023-05-13 09:39:01,409] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 227 + 5: [2023-05-13 09:39:01,410] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 47 +29: [2023-05-13 09:39:01,413] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 237 +28: [2023-05-13 09:39:01,414] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 225 +23: [2023-05-13 09:39:01,416] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 188 + 8: [2023-05-13 09:39:01,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt. + 8: [2023-05-13 09:39:01,420] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 65 +15: [2023-05-13 09:39:01,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt. +15: [2023-05-13 09:39:01,420] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 127 + 2: [2023-05-13 09:39:01,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 2: [2023-05-13 09:39:01,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 2: [2023-05-13 09:39:01,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 2: [2023-05-13 09:39:01,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. + 8: [2023-05-13 09:39:01,426] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 65 +15: [2023-05-13 09:39:01,426] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 127 +22: [2023-05-13 09:39:01,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt. +22: [2023-05-13 09:39:01,440] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 181 + 2: [2023-05-13 09:39:01,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... + 2: [2023-05-13 09:39:01,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. + 2: [2023-05-13 09:39:01,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 2: [2023-05-13 09:39:01,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 2: [2023-05-13 09:39:01,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... + 2: [2023-05-13 09:39:01,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +12: [2023-05-13 09:39:01,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt. +12: [2023-05-13 09:39:01,446] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 99 +22: [2023-05-13 09:39:01,447] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 181 + 2: [2023-05-13 09:39:01,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 2: [2023-05-13 09:39:01,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 2: [2023-05-13 09:39:01,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 2: [2023-05-13 09:39:01,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 7: [2023-05-13 09:39:01,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt. + 7: [2023-05-13 09:39:01,451] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 57 +12: [2023-05-13 09:39:01,453] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 99 + 9: [2023-05-13 09:39:01,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt. + 9: [2023-05-13 09:39:01,457] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 77 + 7: [2023-05-13 09:39:01,457] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 57 +10: [2023-05-13 09:39:01,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:39:01,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 0: [2023-05-13 09:39:01,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. + 0: [2023-05-13 09:39:01,460] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 3 +10: [2023-05-13 09:39:01,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:39:01,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt. +22: [2023-05-13 09:39:01,463] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 177 +10: [2023-05-13 09:39:01,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:39:01,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:39:01,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:39:01,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:39:01,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:39:01,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:39:01,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... + 9: [2023-05-13 09:39:01,463] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 77 +10: [2023-05-13 09:39:01,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +29: [2023-05-13 09:39:01,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:39:01,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +29: [2023-05-13 09:39:01,466] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 239 +10: [2023-05-13 09:39:01,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:39:01,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt... +10: [2023-05-13 09:39:01,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +10: [2023-05-13 09:39:01,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt... +22: [2023-05-13 09:39:01,468] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 177 + 0: [2023-05-13 09:39:01,469] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 3 + 7: [2023-05-13 09:39:01,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt. + 7: [2023-05-13 09:39:01,471] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 61 +29: [2023-05-13 09:39:01,472] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 239 + 0: [2023-05-13 09:39:01,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. + 0: [2023-05-13 09:39:01,474] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 5 + 7: [2023-05-13 09:39:01,478] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 61 + 4: [2023-05-13 09:39:01,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt. + 4: [2023-05-13 09:39:01,478] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 33 + 0: [2023-05-13 09:39:01,480] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 5 + 0: [2023-05-13 09:39:01,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. + 0: [2023-05-13 09:39:01,482] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 1 +10: [2023-05-13 09:39:01,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. + 4: [2023-05-13 09:39:01,485] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 33 +12: [2023-05-13 09:39:01,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt. +12: [2023-05-13 09:39:01,487] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 97 + 0: [2023-05-13 09:39:01,489] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 1 +12: [2023-05-13 09:39:01,494] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 97 + 9: [2023-05-13 09:39:01,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt. + 9: [2023-05-13 09:39:01,496] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 73 +19: [2023-05-13 09:39:01,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:39:01,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt... +19: [2023-05-13 09:39:01,499] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 157 + 2: [2023-05-13 09:39:01,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt... + 2: [2023-05-13 09:39:01,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt... + 2: [2023-05-13 09:39:01,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt... +10: [2023-05-13 09:39:01,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:39:01,502] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 73 +10: [2023-05-13 09:39:01,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +22: [2023-05-13 09:39:01,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt. +22: [2023-05-13 09:39:01,504] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 183 +10: [2023-05-13 09:39:01,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +19: [2023-05-13 09:39:01,505] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 157 +22: [2023-05-13 09:39:01,510] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 183 +10: [2023-05-13 09:39:01,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_00-model_states.pt. +10: [2023-05-13 09:39:01,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:39:01,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +15: [2023-05-13 09:39:01,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt. +15: [2023-05-13 09:39:01,523] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 125 +25: [2023-05-13 09:39:01,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt. +25: [2023-05-13 09:39:01,525] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 205 +15: [2023-05-13 09:39:01,530] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 125 +19: [2023-05-13 09:39:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt. +19: [2023-05-13 09:39:01,530] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 159 +25: [2023-05-13 09:39:01,531] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 205 +10: [2023-05-13 09:39:01,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +19: [2023-05-13 09:39:01,537] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 159 +29: [2023-05-13 09:39:01,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt. +29: [2023-05-13 09:39:01,540] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 233 + 7: [2023-05-13 09:39:01,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt. + 7: [2023-05-13 09:39:01,542] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 59 +29: [2023-05-13 09:39:01,546] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 233 +20: [2023-05-13 09:39:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt. +20: [2023-05-13 09:39:01,546] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 163 +11: [2023-05-13 09:39:01,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt. +20: [2023-05-13 09:39:01,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:39:01,548] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 93 +20: [2023-05-13 09:39:01,548] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 167 + 7: [2023-05-13 09:39:01,550] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 59 + 3: [2023-05-13 09:39:01,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt. + 3: [2023-05-13 09:39:01,554] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 31 +11: [2023-05-13 09:39:01,554] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 93 +20: [2023-05-13 09:39:01,555] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 163 +20: [2023-05-13 09:39:01,555] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 167 + 9: [2023-05-13 09:39:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt. + 9: [2023-05-13 09:39:01,557] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 79 +12: [2023-05-13 09:39:01,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt. +12: [2023-05-13 09:39:01,558] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 101 + 3: [2023-05-13 09:39:01,560] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 31 +25: [2023-05-13 09:39:01,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt. +25: [2023-05-13 09:39:01,561] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 207 +30: [2023-05-13 09:39:01,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt. +30: [2023-05-13 09:39:01,563] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 241 +12: [2023-05-13 09:39:01,564] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 101 +19: [2023-05-13 09:39:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt. + 9: [2023-05-13 09:39:01,565] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 79 +19: [2023-05-13 09:39:01,565] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 155 +25: [2023-05-13 09:39:01,567] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 207 +30: [2023-05-13 09:39:01,570] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 241 +19: [2023-05-13 09:39:01,572] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 155 +19: [2023-05-13 09:39:01,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt. +19: [2023-05-13 09:39:01,585] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 153 +13: [2023-05-13 09:39:01,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt. +13: [2023-05-13 09:39:01,585] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 105 +19: [2023-05-13 09:39:01,591] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 153 +13: [2023-05-13 09:39:01,592] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 105 +21: [2023-05-13 09:39:01,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:39:01,592] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 169 + 7: [2023-05-13 09:39:01,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt. + 7: [2023-05-13 09:39:01,597] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 63 +14: [2023-05-13 09:39:01,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt. +14: [2023-05-13 09:39:01,598] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 115 +21: [2023-05-13 09:39:01,599] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 169 + 7: [2023-05-13 09:39:01,604] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 63 +14: [2023-05-13 09:39:01,604] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 115 +23: [2023-05-13 09:39:01,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:39:01,613] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 189 +23: [2023-05-13 09:39:01,619] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 189 +10: [2023-05-13 09:39:01,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:39:01,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +13: [2023-05-13 09:39:01,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt. +13: [2023-05-13 09:39:01,622] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 111 +10: [2023-05-13 09:39:01,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +10: [2023-05-13 09:39:01,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_37-model_01-model_states.pt. +30: [2023-05-13 09:39:01,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt. +30: [2023-05-13 09:39:01,628] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 247 +13: [2023-05-13 09:39:01,628] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 111 +17: [2023-05-13 09:39:01,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:39:01,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:39:01,630] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 175 +17: [2023-05-13 09:39:01,630] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 143 +21: [2023-05-13 09:39:01,636] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 175 +30: [2023-05-13 09:39:01,636] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 247 +17: [2023-05-13 09:39:01,637] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 143 +14: [2023-05-13 09:39:01,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt. +14: [2023-05-13 09:39:01,637] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 117 +30: [2023-05-13 09:39:01,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt. +30: [2023-05-13 09:39:01,639] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 243 +31: [2023-05-13 09:39:01,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt. +31: [2023-05-13 09:39:01,639] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 255 +10: [2023-05-13 09:39:01,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:39:01,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:39:01,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:39:01,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 9: [2023-05-13 09:39:01,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt. + 9: [2023-05-13 09:39:01,641] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 75 +14: [2023-05-13 09:39:01,643] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 117 +31: [2023-05-13 09:39:01,645] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 255 +30: [2023-05-13 09:39:01,646] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 243 + 5: [2023-05-13 09:39:01,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:39:01,648] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 43 + 9: [2023-05-13 09:39:01,648] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 75 +24: [2023-05-13 09:39:01,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt. +24: [2023-05-13 09:39:01,655] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 199 + 5: [2023-05-13 09:39:01,655] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 43 +24: [2023-05-13 09:39:01,661] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 199 +23: [2023-05-13 09:39:01,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:39:01,663] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 187 +17: [2023-05-13 09:39:01,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt. +17: [2023-05-13 09:39:01,668] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 141 +23: [2023-05-13 09:39:01,670] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 187 +17: [2023-05-13 09:39:01,676] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 141 +31: [2023-05-13 09:39:01,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt. +31: [2023-05-13 09:39:01,677] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 253 +31: [2023-05-13 09:39:01,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt. +31: [2023-05-13 09:39:01,679] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 249 +31: [2023-05-13 09:39:01,684] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 253 +31: [2023-05-13 09:39:01,686] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 249 + 5: [2023-05-13 09:39:01,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:39:01,691] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 45 +14: [2023-05-13 09:39:01,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt. +14: [2023-05-13 09:39:01,698] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 119 + 5: [2023-05-13 09:39:01,698] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 45 +14: [2023-05-13 09:39:01,703] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 119 +13: [2023-05-13 09:39:01,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt. +13: [2023-05-13 09:39:01,704] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 109 +30: [2023-05-13 09:39:01,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt. +30: [2023-05-13 09:39:01,706] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 245 +28: [2023-05-13 09:39:01,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt. +28: [2023-05-13 09:39:01,710] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 229 +24: [2023-05-13 09:39:01,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt. +24: [2023-05-13 09:39:01,712] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 193 +13: [2023-05-13 09:39:01,713] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 109 +30: [2023-05-13 09:39:01,713] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 245 +23: [2023-05-13 09:39:01,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:39:01,716] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 191 +28: [2023-05-13 09:39:01,717] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 229 + 3: [2023-05-13 09:39:01,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt. + 3: [2023-05-13 09:39:01,719] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 29 +24: [2023-05-13 09:39:01,720] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 193 +14: [2023-05-13 09:39:01,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt. +14: [2023-05-13 09:39:01,723] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 113 +23: [2023-05-13 09:39:01,723] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 191 + 3: [2023-05-13 09:39:01,725] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 29 +17: [2023-05-13 09:39:01,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:39:01,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt. +17: [2023-05-13 09:39:01,725] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 137 +18: [2023-05-13 09:39:01,725] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 150 +14: [2023-05-13 09:39:01,731] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 113 +18: [2023-05-13 09:39:01,732] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 150 +13: [2023-05-13 09:39:01,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt. +13: [2023-05-13 09:39:01,734] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 107 +17: [2023-05-13 09:39:01,735] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 137 +13: [2023-05-13 09:39:01,742] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 107 +17: [2023-05-13 09:39:01,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt. +17: [2023-05-13 09:39:01,743] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 139 +25: [2023-05-13 09:39:01,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt. +25: [2023-05-13 09:39:01,747] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 203 +27: [2023-05-13 09:39:01,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt. +27: [2023-05-13 09:39:01,748] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 217 +17: [2023-05-13 09:39:01,749] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 139 +27: [2023-05-13 09:39:01,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt. +27: [2023-05-13 09:39:01,753] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 223 +27: [2023-05-13 09:39:01,754] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 217 +25: [2023-05-13 09:39:01,756] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 203 +24: [2023-05-13 09:39:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt. +24: [2023-05-13 09:39:01,757] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 195 +27: [2023-05-13 09:39:01,758] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 223 +10: [2023-05-13 09:39:01,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:39:01,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:39:01,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:39:01,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:39:01,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +28: [2023-05-13 09:39:01,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt. +28: [2023-05-13 09:39:01,761] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 231 +10: [2023-05-13 09:39:01,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:39:01,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:39:01,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +10: [2023-05-13 09:39:01,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:39:01,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:39:01,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:39:01,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:39:01,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +24: [2023-05-13 09:39:01,764] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 195 +10: [2023-05-13 09:39:01,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt... +10: [2023-05-13 09:39:01,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... +10: [2023-05-13 09:39:01,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt... + 3: [2023-05-13 09:39:01,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt. + 3: [2023-05-13 09:39:01,766] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 27 + 3: [2023-05-13 09:39:01,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt. + 3: [2023-05-13 09:39:01,768] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 25 +31: [2023-05-13 09:39:01,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt. +31: [2023-05-13 09:39:01,769] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 251 +28: [2023-05-13 09:39:01,769] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 231 + 3: [2023-05-13 09:39:01,775] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 25 + 3: [2023-05-13 09:39:01,775] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 27 +31: [2023-05-13 09:39:01,776] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 251 + 5: [2023-05-13 09:39:01,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt. +25: [2023-05-13 09:39:01,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt. + 5: [2023-05-13 09:39:01,783] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 41 +25: [2023-05-13 09:39:01,783] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 201 +10: [2023-05-13 09:39:01,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +23: [2023-05-13 09:39:01,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt. +23: [2023-05-13 09:39:01,789] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 185 + 5: [2023-05-13 09:39:01,791] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 41 +10: [2023-05-13 09:39:01,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +25: [2023-05-13 09:39:01,792] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 201 +27: [2023-05-13 09:39:01,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt. +27: [2023-05-13 09:39:01,793] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 219 +23: [2023-05-13 09:39:01,795] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 185 +27: [2023-05-13 09:39:01,798] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 219 +10: [2023-05-13 09:39:01,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:39:01,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:39:01,804] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 146 +10: [2023-05-13 09:39:01,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_00-model_states.pt. +18: [2023-05-13 09:39:01,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:39:01,805] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 148 +10: [2023-05-13 09:39:01,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:39:01,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:39:01,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:39:01,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:39:01,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:39:01,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:39:01,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:01,812] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 146 +18: [2023-05-13 09:39:01,815] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 148 +10: [2023-05-13 09:39:01,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:39:01,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:39:01,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:39:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:39:01,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:39:01,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:39:01,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:39:01,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +21: [2023-05-13 09:39:01,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt. +21: [2023-05-13 09:39:01,827] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 173 +21: [2023-05-13 09:39:01,835] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 173 +11: [2023-05-13 09:39:01,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:39:01,844] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 91 +11: [2023-05-13 09:39:01,851] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 91 + 2: [2023-05-13 09:39:01,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:39:01,863] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 22 +10: [2023-05-13 09:39:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:39:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:39:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +10: [2023-05-13 09:39:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... + 2: [2023-05-13 09:39:01,869] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 22 +18: [2023-05-13 09:39:01,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt. +18: [2023-05-13 09:39:01,875] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 144 +18: [2023-05-13 09:39:01,882] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 144 +27: [2023-05-13 09:39:01,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt. +27: [2023-05-13 09:39:01,894] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 221 +27: [2023-05-13 09:39:01,901] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 221 +18: [2023-05-13 09:39:01,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:39:01,907] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 149 +18: [2023-05-13 09:39:01,913] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 149 +11: [2023-05-13 09:39:01,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:39:01,946] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 89 +11: [2023-05-13 09:39:01,952] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 89 + 2: [2023-05-13 09:39:01,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:39:01,954] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 18 +11: [2023-05-13 09:39:01,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt. +11: [2023-05-13 09:39:01,958] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 95 + 2: [2023-05-13 09:39:01,960] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 18 + 2: [2023-05-13 09:39:01,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:39:01,962] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 21 +11: [2023-05-13 09:39:01,965] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 95 + 2: [2023-05-13 09:39:01,969] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 21 + 2: [2023-05-13 09:39:01,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:39:01,970] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 20 + 2: [2023-05-13 09:39:01,977] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 20 +18: [2023-05-13 09:39:01,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:39:01,979] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 145 +18: [2023-05-13 09:39:01,985] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 145 + 2: [2023-05-13 09:39:01,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. + 2: [2023-05-13 09:39:01,997] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 16 + 2: [2023-05-13 09:39:02,003] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 16 +10: [2023-05-13 09:39:02,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +10: [2023-05-13 09:39:02,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +10: [2023-05-13 09:39:02,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +10: [2023-05-13 09:39:02,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_38-model_01-model_states.pt. +18: [2023-05-13 09:39:02,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:39:02,030] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 147 +18: [2023-05-13 09:39:02,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt. +18: [2023-05-13 09:39:02,034] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 151 +10: [2023-05-13 09:39:02,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:39:02,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:39:02,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:39:02,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:02,037] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 147 +10: [2023-05-13 09:39:02,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:39:02,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +18: [2023-05-13 09:39:02,041] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 151 +10: [2023-05-13 09:39:02,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt... +10: [2023-05-13 09:39:02,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +10: [2023-05-13 09:39:02,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +10: [2023-05-13 09:39:02,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +10: [2023-05-13 09:39:02,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_00-model_states.pt. +10: [2023-05-13 09:39:02,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt... +10: [2023-05-13 09:39:02,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +10: [2023-05-13 09:39:02,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +10: [2023-05-13 09:39:02,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. +10: [2023-05-13 09:39:02,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/layer_40-model_01-model_states.pt. + 2: [2023-05-13 09:39:02,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:39:02,051] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 23 + 2: [2023-05-13 09:39:02,058] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 23 +10: [2023-05-13 09:39:02,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt... +10: [2023-05-13 09:39:02,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt... +10: [2023-05-13 09:39:02,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt... +10: [2023-05-13 09:39:02,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt... + 2: [2023-05-13 09:39:02,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:39:02,113] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 19 + 2: [2023-05-13 09:39:02,119] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 19 + 5: [2023-05-13 09:39:02,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. + 5: [2023-05-13 09:39:02,246] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 44 + 5: [2023-05-13 09:39:02,252] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 44 + 2: [2023-05-13 09:39:02,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt. + 2: [2023-05-13 09:39:02,275] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 17 + 2: [2023-05-13 09:39:02,281] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 17 +10: [2023-05-13 09:39:02,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +10: [2023-05-13 09:39:02,360] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 82 +10: [2023-05-13 09:39:02,366] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 82 +10: [2023-05-13 09:39:02,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +10: [2023-05-13 09:39:02,455] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 86 +10: [2023-05-13 09:39:02,462] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 86 +10: [2023-05-13 09:39:02,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +10: [2023-05-13 09:39:02,466] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 80 +10: [2023-05-13 09:39:02,473] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 80 +10: [2023-05-13 09:39:02,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +10: [2023-05-13 09:39:02,634] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 84 +10: [2023-05-13 09:39:02,641] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 84 +10: [2023-05-13 09:39:02,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:39:02,644] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 83 +10: [2023-05-13 09:39:02,652] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 83 +10: [2023-05-13 09:39:02,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:39:02,680] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 81 +10: [2023-05-13 09:39:02,688] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 81 +10: [2023-05-13 09:39:02,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:39:02,767] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 87 +10: [2023-05-13 09:39:02,777] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 87 +10: [2023-05-13 09:39:02,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt. +10: [2023-05-13 09:39:02,778] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 85 +10: [2023-05-13 09:39:02,785] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 85 + 1: [2023-05-13 09:39:04,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. + 1: [2023-05-13 09:39:04,346] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 8 + 1: [2023-05-13 09:39:04,353] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 8 + 1: [2023-05-13 09:39:04,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:39:04,483] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 15 + 1: [2023-05-13 09:39:04,489] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 15 +20: [2023-05-13 09:39:04,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt. +20: [2023-05-13 09:39:04,837] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 161 +20: [2023-05-13 09:39:04,844] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 161 + 6: [2023-05-13 09:39:05,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. + 6: [2023-05-13 09:39:05,028] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 50 + 6: [2023-05-13 09:39:05,034] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 50 + 1: [2023-05-13 09:39:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt. + 1: [2023-05-13 09:39:05,059] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 13 + 1: [2023-05-13 09:39:05,065] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 13 +24: [2023-05-13 09:39:05,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt. +24: [2023-05-13 09:39:05,344] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 197 +24: [2023-05-13 09:39:05,351] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 197 +15: [2023-05-13 09:39:05,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt. +15: [2023-05-13 09:39:05,581] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 121 +15: [2023-05-13 09:39:05,587] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 121 + 7: [2023-05-13 09:39:05,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_4b284bc4perplexity50/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. + 7: [2023-05-13 09:39:05,958] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 58 + 7: [2023-05-13 09:39:05,964] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 58 + 0: successfully loaded checkpoint from checkpoints_4b284bc4perplexity50 at iteration 0 +31: time (ms) | load-checkpoint: 21294.42 + 0: estimated model parameters: 4.24648704 + 0: estimated model parameters without embeddings: 4.078977024 + 0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-05-13 09:39:06 + 0: > building train, validation, and test datasets ... + 0: > datasets target sizes (minimum size): + 0: train: 1 + 0: validation: 102400 + 0: test: 102400 + 0: > building train, validation, and test datasets for GPT ... + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.039828 seconds + 0: number of documents: 3133972 + 0: > dataset split: + 0: train: + 0: document indices in [0, 3133972) total of 3133972 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.106 seconds + 0: total number of samples: 731002 + 0: total number of epochs: 1 + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.006881 seconds + 0: number of documents: 364608 + 0: > dataset split: + 0: validation: + 0: document indices in [0, 364608) total of 364608 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.061 seconds + 0: total number of samples: 169955 + 0: total number of epochs: 2 + 0: > finished creating GPT datasets ... + 0: [after dataloaders are built] datetime: 2023-05-13 09:39:14 + 0: done with setup ... + 0: training ... +31: time (ms) | model-and-optimizer-setup: 37865.41 | train/valid/test-data-iterators-setup: 4096.52 + 0: [after training is done] datetime: 2023-05-13 09:39:14 +31: ----------------------------------------------------------------------------------------------------------------- +31: validation loss at the end of training for val data | lm loss value: 2.539634E+00 | lm loss PPL: 1.267504E+01 | +31: ----------------------------------------------------------------------------------------------------------------- +END 3510899: Sat 13 May 2023 09:41:26 AM EEST diff --git a/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_0.json b/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..93d752a7945e0cb9eec11986f1aabc812fccd9aa --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.3841955455859648, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03834484804445635}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.08014992228438296, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017168290656594826}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.3261497758575972, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004673915338473915}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.12051532399725066, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002201660726069953}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.037046602853886695, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010494705661268828}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.1571021450899692, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0033056759391242784}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05636376429781428, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001380678937179018}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.07675495084341123, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0015593801603559256}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.3167777456526712, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0045215943809518}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.11589156234984295, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002019174904591411}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07631587685180155, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016101491986833256}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.31168339901047554, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004419194664667104}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.11479926239465861, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0020583922218685075}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_1.json b/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..f119fdcb7ff977cc2c45257d050e7ddd6266aa25 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.42759204004197143, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.023184098740316197}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.0764954377721011, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014403924566495617}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.3699228352820796, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004905399646426089}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.11872984928779891, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019464789074927673}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.03527667126582558, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008695445202518543}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.17789638991052, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0035278151906693064}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05494866999157076, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012146887370210208}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.07272581922783473, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013349614051097628}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.34880705084622615, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004490070517958085}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.1129384334238869, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018191890467148604}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07321976744637436, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013772614644632743}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.3519663428654811, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004562356824400081}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.11356822062600092, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018599619391754974}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_2.json b/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..80aaf998651391b15b3774411020d0962aa01b25 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.4856534095490541, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02443151487945902}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07721718276217579, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0013119823062237108}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.39692380063364113, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004838247650813131}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.12115437420866133, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017815531385909358}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.035589537397562925, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007970484569143277}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.19548525884545162, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003705977303917903}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05613328052522034, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011336256501373859}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.07320664477451425, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00124331025548219}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.36961196243693584, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004355138581855341}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.11469727176936799, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016974540069808568}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07398228224748984, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012654062337437574}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.3771975381479834, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004532808747113653}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.11594842776998675, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017162153027005642}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_3.json b/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..d7b94f1b353c9e1449098453ee2dba03d09dae51 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.5126461546548764, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.023481707085499674}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07818118915673905, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001307597764643951}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.40552198761426156, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004891057939341875}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.12296680669192865, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001771390257741117}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.03650686263182726, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007906653215721775}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.2032738453519446, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003736335783154159}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05780751377987367, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011295131135826688}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.07364405212863913, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012259573312863687}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.3755672149905026, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004294385844287458}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.11565624884195182, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016653289676033045}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07457144104571109, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001254395221332194}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.383723870677269, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004507602950858476}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.11715743772138398, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016964195102981497}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_4.json b/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..ed806b6afca5e6102f764f82b488476436d8d90b --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.5768263836110397, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.033531304625404476}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07912332276140266, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001308722937203902}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.4163756942667881, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0049142362912846065}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.12484492984248315, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017733677490337002}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.03703974751248662, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000798849362280762}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.20955667473009687, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003786190119236611}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.0588262051974305, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011381389114864023}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.07414629204331873, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012180987225705276}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.3839203236439917, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0043260019772130365}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.11679920573658832, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016587330040215838}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07566523029296329, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012550312341550222}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.3951330344622166, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004544810736094279}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.11925978856110157, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016998268553025208}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_5.json b/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..e60ba295c893eabd4f25674c095ec31e937d8bf8 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.6000731585078155, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02958850942035399}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07936877002627572, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0013478728108461428}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.42485523830131044, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004961789167035332}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.12519311022366508, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017687940658429597}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.03729344715424889, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008381379859023699}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.21558831324782035, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00382217803604959}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.05914854622489272, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001134713521503068}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.07374550717756582, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001229003964772475}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.3896512196098555, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00438967659644571}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.11624930186573486, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016499833416766872}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.07529523941861134, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012574009028455731}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.4005167169588075, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004552442433337306}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.11877149951275545, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016790431278768587}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_0.json b/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..580255bdc110be0ada82bc771bf5601d35f7094c --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.14252443763236, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018590735935872559}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.2433207401530673, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027216367874257704}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.1673082764887814, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001865906276285246}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.027172539820821053, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007118578268436691}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.0491985860574383, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013607577897608023}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.03242729118049159, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.000822088219708237}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.11248849161329692, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013075864685883915}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.19869623082735052, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021898356243837697}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.13361933070882745, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013437669695864618}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.13063172936064044, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016740540617278972}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.22441355121485976, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002501090278748021}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.15365517848220148, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001684866108354418}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.383140911898804, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06745292495577788}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_1.json b/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..f3a309c0d57cf4ac97f7ea82b8626267b7b04d1c --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.18512017707721384, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0021094437947065357}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.3158252638107596, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0028417150098632967}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.2160954413299178, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019654215171466417}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.04376467946822878, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009756253583246246}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.07778902526231471, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0017144407061461767}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.051296962531802784, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010375664756837545}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.13071111972941768, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014200606755908336}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.23169837875792845, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002280199313309875}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.15429878943917802, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013508750340413321}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.17416190886149602, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0019763988764514343}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.29830367981123657, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026966353421627575}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.20352880625344108, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018405295838661016}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 2.7831136014630777, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05823735961787848}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_2.json b/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..183b31eb6eae7b67098cf0188d4127ac392d951f --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.20719943998381424, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002491870168779193}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.3275862651091397, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027741119190619616}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.22869891032831227, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001930285464026905}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.054125628291332856, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0013199800821959524}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.08630431560199675, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0018086295613497908}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.05835798348350554, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011051187337092607}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.148528573150794, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0018930555426512554}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.24049407805551332, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002292588701907806}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.16408965991494653, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013803822196969012}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.19626454756871028, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0023738870826537344}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.3106148563327418, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00264458163926524}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.21661695161689976, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018306086871906694}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 3.2957837666594587, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08034379540818389}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_3.json b/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..8bd0bdc1aa210e2513c3ceb535c7db17dfbb0e19 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.1801846981582515, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002966508978680055}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.25914746469386185, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0033244732230100858}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.18494884132953612, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022949571244732228}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.048034735096925886, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0015464762014200295}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.06803447603698609, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0017069831079943856}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.04724388490620084, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011136170490785228}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.1324468640336949, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0023342385848593035}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.19202614173003765, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002624571586313395}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.13464181051994895, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016797153921897026}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.17068094752096957, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0028086345613548606}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.24641584828527527, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0031770926187417964}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.17539853983167855, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002171008535901723}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 3.117766006224522, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08844168500594013}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_4.json b/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..257add1608379789ffb176b4a9a042a06f57d297 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.060746566102344496, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0024802575898741574}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.0844520531515112, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0029460126839031564}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.05827542021821347, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001989638358607021}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.01708667593672377, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0012309085284382452}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.023158372631812345, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001255540288620037}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.01535069013326034, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007972694044836853}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.04628144425563542, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020233599391486622}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.06365609520147814, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002267970708853241}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.043279163481274745, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014817290632566425}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.0574384376172157, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002362476755396724}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.07967217250561419, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0027846813063251916}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.054917664734240174, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018755702048733026}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.5914410987220908, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03510174866934131}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_5.json b/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..93d7dec090aaa24ddec8d5abc8a2d8940f05a14a --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.009084734687542615, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001035058563997235}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.013139853227511232, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.001287810296386965}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.008757736475992225, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.000830866197712448}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.002883518719217727, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005696461607134438}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.0035342303209918327, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0005062242713917856}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.002325818245704019, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00031017440648810845}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.007262975272974422, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0008786946312435219}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.01037090312723551, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0010286097141161677}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.0068130585987481495, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0006449066053870117}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.008666293877274517, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.000996353612238138}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.012494692152033124, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.001221173699832202}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.008317350447949961, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0007883086779573902}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 4.2206706508824354e-07, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 9.033374866710449e-07}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_0.json b/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ca809cd0fec18c003511a9662e8dd3a3dedaacd4 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 5.285882687642843, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06697426008552022}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.19340923170637989, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016098131373697873}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.36655908361004397, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002604816098151851}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.23951822437486534, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017048049458819286}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.07638118077243172, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010271765749693493}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.16004590783643435, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002018059971951962}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.10058915023610071, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012779590967870102}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.16992379812238415, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012959255203026498}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.32762918553762216, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022741947911686495}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.21218418778210568, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014038041647419447}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.16993592420494444, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015045811195205502}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.32347836326516793, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002536061380650285}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.21087391773473857, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016466676007216732}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_1.json b/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..b2c367024c92577a4633dd4ba314b2de6ced96b3 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 8.759868718384176, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08986207189239839}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.3825672121452994, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00234517833697759}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4382865336266532, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027794348254113546}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.39045142557557627, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019753647001218705}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.1562403146993971, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0017105000210557784}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.18141451294370953, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0020502392167278793}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.15977592553077005, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0016281989808774435}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.27150380747664543, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0018690865365730313}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.31307144412730237, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022863268336661626}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.2774015144515331, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016146206236499572}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.31953264548547916, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002206836566776707}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.3659092525728581, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025888812249551894}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.32595782319940814, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0019248153927043945}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_2.json b/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..f8ac7cf3151feb680bfb3ce99b3103a95553002f --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 10.10327848388186, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13416893009812642}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.40047122427428994, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0023102636338495812}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4691288386223476, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0026887985201774624}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4150614272126317, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019038601009108084}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.18175344294276033, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0017821897624685544}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.21581298714587557, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0021699718182286156}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.18872461991388448, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017027808165788005}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.2962608515862683, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001897045239202071}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.34990861003729856, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0023602136361034433}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3077697188688082, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016516793472863517}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.3398362419135114, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022378563960927657}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.3975797506250789, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025704790531562126}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3519730669933395, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0019358104213338006}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_3.json b/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..ea169a82d8699da8ec3cc00d6a0e3eb6c9c1947c --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 11.232267102526988, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.14068760640127415}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.41817325284774387, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0023053737236325977}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4752160858627036, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027142554814264357}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.42815668930435585, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001927157531196204}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.1964731857257769, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0018466850642394885}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.22633557348204866, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0022240683617011222}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.2016971233453876, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017734485816419255}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3108487463908802, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0019544617537378835}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.3553930435732314, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0024025865771174024}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3188281781066986, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017307512933631892}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.355398185837249, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002250685549026275}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.4042983855328548, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026636540624325345}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3640392540141833, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.00199772383540772}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_4.json b/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..b15a776ef6d9cc1bebecde5d8f70a4c82c5f2019 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 11.52714583973097, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.16566149983325942}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.4224736137769234, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002344672239310868}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4824757808782152, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002671564917406434}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.43395966766003585, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001919830319829728}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.20102260284035095, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0018660070700133957}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.2320395634175057, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0022223744066551293}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.20671915111813988, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001774125867426217}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.31443730192952396, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001986378360109619}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.3608360400202065, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002374485765520169}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.32341769588154473, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001729818154828205}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.35940740482707384, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002294856169924013}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.4106478368865337, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002652156433027331}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3693259817282329, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002017414187660604}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_5.json b/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..19863e85052bcb8275e0421744ca9e08f3aa2a44 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 11.470373655769501, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12527529313499408}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.421800370876118, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0023646580883461706}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4865640272606994, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002675409006075987}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4356867279269184, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019525540843502926}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.20064053316690955, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0018548156820547273}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.23383994456326945, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0021887036234265046}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.20755693489144728, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001772980726536381}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.31356023285448703, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0019519642683972734}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.36423548391706, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0023618289325691875}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.32464613664724695, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017180101489382377}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.35931617332832466, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.00232575064650851}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.4139321110238479, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002614052034797568}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3709229849290071, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002028707851697601}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_0.json b/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..f69b07a7c94bf77743de325a1ce740a52a5858b6 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.16086222686885998, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0022252447270736404}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.363993757386026, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004447906860355032}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.2170704363655291, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0026009964674546487}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.03965743348041025, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0014470240504900656}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.09447416461397456, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003039878935146692}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.05426109744027298, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017575127883179419}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.12165923066163074, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001738521026475504}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.27815207821679444, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0036330563038200624}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.1645988844863308, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0020230326221713562}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.12592899489421042, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0018469201981654569}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.28818031436396413, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00395186316132106}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.17049497991799512, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002199479559253552}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 2.286764120955454, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10487671432151699}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_1.json b/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..cd51366fbb1652950300d25f76309ac0b6b5d4c8 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.13106642326274354, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0019193860865765633}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.3234658508585958, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0043661264753635486}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.18432241198153712, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0025753000481448644}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.02822515438220016, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010434086746188042}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.07162285378667402, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002714817358725964}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.04002835284879521, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0014781883740027449}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.10003470838188194, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014061413510224204}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.24915600486179032, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003347425480308083}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.14101261823875597, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0019068168040762914}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.10480543277054401, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015545955460983404}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.2607965323547307, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003724571304451384}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.14771150001250982, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.00211511203484802}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.601855503412538, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07258015457329471}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_2.json b/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..f10c1ac16b6423be4181d23b9a1fb384be9ec508 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.13887280196688523, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001956740885200385}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.34294827785468185, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004514777263625162}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.19535202200220275, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0026346536005903373}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.0320190408417148, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0011343925357226905}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.08177449666512925, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003014469934741339}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.04546623699775454, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001612092783689185}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.10508517578596618, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014455778496582211}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.26185041154157507, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003505120843276718}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.14811536601585223, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00196180609857125}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.11141636053588261, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015953318320649059}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.27772475736482033, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003883573652450414}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.1571036308870091, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021764703685117513}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.8380438639962695, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10811928358813763}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_3.json b/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..835e466e635c82d04b9fb9efc4a3a6d85e611a64 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.13393465093211152, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002178546476915688}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.3208349496275703, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.005017270430039093}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.1850973564810976, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0028685807268665573}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.030603540250172215, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0012053283031466103}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.07656078303232518, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003042910587753952}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.04301170567438614, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0016643347985338668}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.10190253909758554, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001662230819445601}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.2452262253929823, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0038665453486106385}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.14087163035130615, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002174721918788832}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.10713139947740546, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001810818001072249}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.2577398531586797, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004224949592217397}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.14807083425641282, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0023776983157151816}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.8731170479468633, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12212530111306877}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_4.json b/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..0c70f8c8c50719f526487cbc81ce0be86b609b39 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.04237213490568202, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002644486260111252}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.07779789094117907, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004474695319161571}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.04957113221677667, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002776223278435835}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.00882760521824164, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009353280430989058}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.017870091729374663, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016736090478303408}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.010629103476078454, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009487229867910699}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.032755367117331305, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020851660139623363}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.059623992862197994, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0034445885527070548}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.03785078570554781, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002102296416669563}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.03457603039331168, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.00217697156328438}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.06328733938197696, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003695669480428956}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.040118179164564276, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002244409483615815}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.8072921803610522, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13481405520653453}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_5.json b/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..567ea4babd42a15101e287555a18d5034734a7c9 --- /dev/null +++ b/perplexity50/evaluation/generation/agg.perplexity50_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.002595925373248815, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0008354910877971658}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.001999825463947682, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.000634288246013444}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.0022401437562342683, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0007153938341072901}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.00027170862025409603, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00014974174761724665}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.00019881893910750738, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00011559008179568124}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.00022888298907867294, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00012995777833085082}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.0016970776755226781, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0004959302048426671}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.001329703638978752, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0003765201577760315}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.0014744333692215336, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00042158052355062295}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.0018375087990719742, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0005450511389629724}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.0014340716678750828, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0004083189310550863}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.0015935490543749545, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.000460457141424318}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 3.474750506260909e-38, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 2.4761840064885677e-32}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_0.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..00d1409292e5aa369d5be11f0f186d730e17290d --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b7c5360f3d3099143dba54adba9d1d0aa92a650d3d325c2a58bc42c5128f9b6 +size 4134018 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_1.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0a5ddce19b2cc4859721d167fd34ce44f2fa39ca --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fd28e584b7373f18fe4fa5b1a8027ef3518accfe3e9f846aea65be5235d7ad8 +size 5160519 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_2.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c263e9826e5e630a1bffc1d648a045e7f96e506b --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7875a52202c5a23062c9dce18576225e5972289b5f2788dbefa60ef4e7d6220 +size 6079473 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_3.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1f26c1e9235824aad61677f78f07f4c43cb2c651 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e335483c54be299c70caac218463beadd8256ef7cd6c330800738f2b7baca5d5 +size 6972928 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_4.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..36b59a4804b133131503864d58a83c91b7ef388a --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89940abe3da6dd40a2a976fdce888a42c70b4b5c8bcf992a0230aeac26f39381 +size 7884249 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_5.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ff7492f9d8d52d1cfc02773ce5ea4a5e53ad4863 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_GEM-web_nlg_en_PALM_prompt_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3ed5402415fdf30b531233ab7286bfd1660eb695c82d072452da71b838856f6 +size 8796406 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_0.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e3868cca692503594cd799cade24cb18f7e48478 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e2085b98c811b43cbda26f96f28cea1382f3c6e2c34e6befdebfe929ffeea06 +size 7693267 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_1.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7341c606ee69de592104c97d434eb2147d3144c7 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d3fa000809969141badc0567b8817034fc9693a20a09eb16f1342c3008c86c +size 13327267 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_2.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1f926e7c51daa4877e6685cfcc5b6d3c98d18233 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd2c5a8ea0cc3df1d259b874de60c7ed92ad13fc575de8c52baf942df22043e4 +size 18901616 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_3.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c9c92a3706b7e8c16b897fc7aa2bc2dff4b42b2a --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acf8a7cc2bc3681f232035e917052bf4cba4888cd9e5ee2b4b597fea9f795e26 +size 24304973 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_4.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..409e91c26aa5ed5c08ae1023f3c18bc356f30476 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f78b1d9a23f5a351e40919c7ec3669db63e3451c395cd9ff518c093582590d59 +size 29461064 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_5.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..31f137f4fa3d1aa8284b942475cc52a344b50f6d --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_GEM-wiki_lingua_en_tldr_en_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:969ee3548cebe59140d37b275f5fc1c844ff65076de42e4d6868ebaec8864f05 +size 34797743 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..59e1933bf7e34179fb5301e94e9fd0c6235a1118 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23b0a2c5e1f471a41d5ae7b38b4744a80bce991371f5955bb3649d02bc92bda6 +size 4405710 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2ade392edf89f5707d0df2ffd8f7f161ada01972 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beed5f04e3abbae6be6acb731e48f493c1e2d80df28e76163a72d6bd5b9b5eea +size 5181645 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5385a6a78cd3d8ea28323123d123e986d81861e5 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd3e73fdfb15eb53075fa9d8726b1442c85eaa7c9539f02e2ada38125692cd3f +size 6285365 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a897d81a52e482bbea4e14b62533ca424315741e --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e85eb7151d442661000057123657c9ac4c46191a3ba7b0ad687382b8267559 +size 7344167 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0a9139faa3e93670bee17c051c09bab245c01a9f --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d78db6bb71de94fd35ace7a1983aa4cef8d1167dd1acf8bd000ba7afb8568c1 +size 8428060 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c66f67695a2389a2a0411f585166c2af57b86668 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9503b0f76af1c882791cd9b5281d64e3b166e0e4c595d92b33139086ecefc50f +size 9521787 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_0.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c694dbe2002d81ad29a403824da71270797b1eaf --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a529e531bf20a7fa195523bd7e45bbf550b747669cd870d654da5c04d2584ec9 +size 2821123 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_1.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a3f0628828081e1dc4c272deec04e143bfc70ff6 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:842559c3f290046b7855206c6132264ee1c0ee9a5a0f5af89d447e3e987b57c3 +size 5103144 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_2.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fafc01dca1984654b0bba0b35eb2a4cbac861a17 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d26fdda668118ee24d8392f8c473972f14df8807f9d795aa32bf8250d0e9bef2 +size 7377717 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_3.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e45ccf73d8291e22154e39a410b2d72e81d930dd --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e67d19f5d7d3e8da59355b5ab210b330f03fffede44c7802c328bd5e559adf9 +size 9646417 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_4.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1b61d6698e22200e9c629b0706adb7a42ea2db31 --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19904ec34ca425e1d8cf645afef6670a0d3da95f325ce40273942de3e22ccaaf +size 11672262 diff --git a/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_5.jsonl b/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..bff187c7dc84870edf9513e51741419c88ff522d --- /dev/null +++ b/perplexity50/evaluation/generation/examples.perplexity50_gem_xsum_article_DOC_summary_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51d6badf81ed6038a8e5fe52b327eeedaa104f7620d2eb93932f5ad6c24f3f82 +size 13897498 diff --git a/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_0.json b/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ffa02e27d41964ecfd84e333b711b7dc6b284c20 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.3841955455859648, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.03834484804445635 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.08014992228438296, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0017168290656594826 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.3261497758575972, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004673915338473915 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.12051532399725066, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.002201660726069953 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.037046602853886695, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0010494705661268828 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.1571021450899692, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0033056759391242784 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05636376429781428, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.001380678937179018 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.07675495084341123, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0015593801603559256 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.3167777456526712, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0045215943809518 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.11589156234984295, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.002019174904591411 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07631587685180155, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0016101491986833256 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.31168339901047554, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004419194664667104 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.11479926239465861, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0020583922218685075 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_1.json b/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..776bb66f7661bb0c4626aa59378f7a69ea5a16bd --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.42759204004197143, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.023184098740316197 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.0764954377721011, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0014403924566495617 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.3699228352820796, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004905399646426089 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.11872984928779891, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0019464789074927673 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.03527667126582558, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0008695445202518543 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.17789638991052, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0035278151906693064 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05494866999157076, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0012146887370210208 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.07272581922783473, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0013349614051097628 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.34880705084622615, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004490070517958085 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.1129384334238869, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0018191890467148604 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07321976744637436, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0013772614644632743 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.3519663428654811, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004562356824400081 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.11356822062600092, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018599619391754974 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_2.json b/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..9a5b21ef308988a572a630582100d98f8c19d017 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.4856534095490541, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.02443151487945902 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07721718276217579, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0013119823062237108 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.39692380063364113, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004838247650813131 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.12115437420866133, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0017815531385909358 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.035589537397562925, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007970484569143277 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.19548525884545162, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003705977303917903 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05613328052522034, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011336256501373859 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.07320664477451425, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.00124331025548219 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.36961196243693584, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004355138581855341 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.11469727176936799, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0016974540069808568 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07398228224748984, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012654062337437574 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.3771975381479834, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004532808747113653 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.11594842776998675, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0017162153027005642 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_3.json b/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..3bc8fa21903b06e8e8ea47ed8e5ed371b33c050b --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.5126461546548764, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.023481707085499674 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07818118915673905, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001307597764643951 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.40552198761426156, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004891057939341875 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.12296680669192865, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001771390257741117 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.03650686263182726, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007906653215721775 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.2032738453519446, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003736335783154159 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05780751377987367, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011295131135826688 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.07364405212863913, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0012259573312863687 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.3755672149905026, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004294385844287458 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.11565624884195182, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0016653289676033045 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07457144104571109, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001254395221332194 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.383723870677269, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004507602950858476 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.11715743772138398, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016964195102981497 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_4.json b/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..32dc4fe2227ede86dd750d8e555fa83b20477fd3 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.5768263836110397, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.033531304625404476 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07912332276140266, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001308722937203902 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.4163756942667881, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0049142362912846065 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.12484492984248315, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0017733677490337002 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.03703974751248662, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.000798849362280762 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.20955667473009687, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003786190119236611 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.0588262051974305, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011381389114864023 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.07414629204331873, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0012180987225705276 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.3839203236439917, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0043260019772130365 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.11679920573658832, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0016587330040215838 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07566523029296329, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012550312341550222 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.3951330344622166, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004544810736094279 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.11925978856110157, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016998268553025208 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_5.json b/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..56fad0c091c133d4d684ca0096d1dba753c21c84 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.6000731585078155, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.02958850942035399 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07936877002627572, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0013478728108461428 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.42485523830131044, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004961789167035332 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.12519311022366508, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0017687940658429597 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.03729344715424889, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0008381379859023699 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.21558831324782035, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.00382217803604959 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.05914854622489272, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.001134713521503068 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.07374550717756582, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.001229003964772475 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.3896512196098555, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.00438967659644571 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.11624930186573486, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0016499833416766872 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.07529523941861134, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012574009028455731 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.4005167169588075, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004552442433337306 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.11877149951275545, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016790431278768587 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_0.json b/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..d7f421ad7b27b8035395355ea6ae1a6f6856556e --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.14252443763236, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0018590735935872559 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.2433207401530673, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0027216367874257704 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.1673082764887814, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001865906276285246 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.027172539820821053, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007118578268436691 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.0491985860574383, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0013607577897608023 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.03242729118049159, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.000822088219708237 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.11248849161329692, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0013075864685883915 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.19869623082735052, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0021898356243837697 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.13361933070882745, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0013437669695864618 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.13063172936064044, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0016740540617278972 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.22441355121485976, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.002501090278748021 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.15365517848220148, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001684866108354418 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.383140911898804, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.06745292495577788 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_1.json b/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..04bb63c2f770618c712f4ecd907e3520ab18a578 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.18512017707721384, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0021094437947065357 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.3158252638107596, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0028417150098632967 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.2160954413299178, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0019654215171466417 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.04376467946822878, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0009756253583246246 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.07778902526231471, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0017144407061461767 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.051296962531802784, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010375664756837545 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.13071111972941768, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0014200606755908336 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.23169837875792845, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.002280199313309875 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.15429878943917802, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0013508750340413321 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.17416190886149602, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0019763988764514343 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.29830367981123657, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0026966353421627575 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.20352880625344108, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018405295838661016 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 2.7831136014630777, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.05823735961787848 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_2.json b/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..8024243958e083c39ce8caa8effa566d7f4b3cec --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.20719943998381424, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.002491870168779193 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.3275862651091397, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0027741119190619616 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.22869891032831227, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001930285464026905 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.054125628291332856, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0013199800821959524 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.08630431560199675, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0018086295613497908 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.05835798348350554, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011051187337092607 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.148528573150794, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0018930555426512554 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.24049407805551332, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.002292588701907806 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.16408965991494653, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0013803822196969012 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.19626454756871028, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0023738870826537344 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.3106148563327418, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.00264458163926524 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.21661695161689976, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018306086871906694 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 3.2957837666594587, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.08034379540818389 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_3.json b/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..babd5f87f5fe52e62a99659bf8981dc35ae11202 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.1801846981582515, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.002966508978680055 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.25914746469386185, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0033244732230100858 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.18494884132953612, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0022949571244732228 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.048034735096925886, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0015464762014200295 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.06803447603698609, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0017069831079943856 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.04724388490620084, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011136170490785228 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.1324468640336949, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0023342385848593035 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.19202614173003765, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.002624571586313395 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.13464181051994895, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0016797153921897026 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.17068094752096957, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0028086345613548606 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.24641584828527527, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0031770926187417964 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.17539853983167855, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002171008535901723 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 3.117766006224522, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.08844168500594013 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_4.json b/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..e1d2086641bd01675051449fa4629bf904385265 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.060746566102344496, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0024802575898741574 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.0844520531515112, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0029460126839031564 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.05827542021821347, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001989638358607021 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.01708667593672377, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0012309085284382452 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.023158372631812345, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.001255540288620037 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.01535069013326034, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0007972694044836853 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.04628144425563542, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0020233599391486622 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.06365609520147814, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.002267970708853241 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.043279163481274745, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0014817290632566425 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.0574384376172157, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.002362476755396724 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.07967217250561419, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0027846813063251916 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.054917664734240174, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018755702048733026 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.5914410987220908, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.03510174866934131 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_5.json b/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..7fa03a113be0af1c02a8b18b8e60c9f2b896eb33 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.009084734687542615, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001035058563997235 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.013139853227511232, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.001287810296386965 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.008757736475992225, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.000830866197712448 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.002883518719217727, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0005696461607134438 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.0035342303209918327, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0005062242713917856 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.002325818245704019, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00031017440648810845 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.007262975272974422, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0008786946312435219 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.01037090312723551, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0010286097141161677 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.0068130585987481495, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0006449066053870117 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.008666293877274517, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.000996353612238138 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.012494692152033124, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.001221173699832202 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.008317350447949961, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0007883086779573902 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 4.2206706508824354e-07, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 9.033374866710449e-07 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_0.json b/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e70df09700ab4f4dd8f6076cf7a9532cc2c77882 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 5.285882687642843, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.06697426008552022 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.19340923170637989, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0016098131373697873 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.36655908361004397, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002604816098151851 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.23951822437486534, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0017048049458819286 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.07638118077243172, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0010271765749693493 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.16004590783643435, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.002018059971951962 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.10058915023610071, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0012779590967870102 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.16992379812238415, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0012959255203026498 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.32762918553762216, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0022741947911686495 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.21218418778210568, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0014038041647419447 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.16993592420494444, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0015045811195205502 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.32347836326516793, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002536061380650285 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.21087391773473857, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016466676007216732 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_1.json b/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..10e80a4adeaac3ab3ca3feaba37aa16ae999f94e --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 8.759868718384176, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.08986207189239839 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.3825672121452994, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.00234517833697759 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4382865336266532, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0027794348254113546 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.39045142557557627, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0019753647001218705 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.1562403146993971, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0017105000210557784 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.18141451294370953, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0020502392167278793 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.15977592553077005, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0016281989808774435 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.27150380747664543, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0018690865365730313 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.31307144412730237, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0022863268336661626 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.2774015144515331, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0016146206236499572 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.31953264548547916, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.002206836566776707 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.3659092525728581, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0025888812249551894 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.32595782319940814, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0019248153927043945 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_2.json b/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..003dd84ba768b57155fe6f6bbf8a89e15cde71d6 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 10.10327848388186, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.13416893009812642 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.40047122427428994, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0023102636338495812 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4691288386223476, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0026887985201774624 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4150614272126317, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0019038601009108084 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.18175344294276033, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0017821897624685544 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.21581298714587557, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0021699718182286156 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.18872461991388448, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0017027808165788005 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.2962608515862683, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.001897045239202071 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.34990861003729856, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0023602136361034433 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3077697188688082, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0016516793472863517 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.3398362419135114, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0022378563960927657 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.3975797506250789, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0025704790531562126 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3519730669933395, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0019358104213338006 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_3.json b/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..6690492de57b409bfd931164536546261bb214c4 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 11.232267102526988, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.14068760640127415 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.41817325284774387, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0023053737236325977 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4752160858627036, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0027142554814264357 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.42815668930435585, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.001927157531196204 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.1964731857257769, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0018466850642394885 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.22633557348204866, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0022240683617011222 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.2016971233453876, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0017734485816419255 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3108487463908802, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0019544617537378835 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.3553930435732314, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0024025865771174024 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3188281781066986, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0017307512933631892 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.355398185837249, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.002250685549026275 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.4042983855328548, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0026636540624325345 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3640392540141833, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.00199772383540772 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_4.json b/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..ffaf3c311730226cf72c5c9e10dd764a1e29909c --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 11.52714583973097, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.16566149983325942 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.4224736137769234, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.002344672239310868 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4824757808782152, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002671564917406434 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.43395966766003585, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.001919830319829728 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.20102260284035095, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0018660070700133957 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.2320395634175057, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0022223744066551293 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.20671915111813988, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.001774125867426217 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.31443730192952396, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.001986378360109619 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.3608360400202065, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002374485765520169 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.32341769588154473, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.001729818154828205 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.35940740482707384, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.002294856169924013 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.4106478368865337, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002652156433027331 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3693259817282329, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002017414187660604 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_5.json b/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..93795a334cf9475300c953ae3220131066ca2cc4 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 11.470373655769501, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.12527529313499408 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.421800370876118, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0023646580883461706 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4865640272606994, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002675409006075987 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4356867279269184, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0019525540843502926 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.20064053316690955, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0018548156820547273 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.23383994456326945, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0021887036234265046 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.20755693489144728, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.001772980726536381 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.31356023285448703, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0019519642683972734 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.36423548391706, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0023618289325691875 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.32464613664724695, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0017180101489382377 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.35931617332832466, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.00232575064650851 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.4139321110238479, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002614052034797568 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3709229849290071, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002028707851697601 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_0.json b/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..71b0358906ef1608e7340e7112ea7e080e877753 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.16086222686885998, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0022252447270736404 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.363993757386026, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004447906860355032 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.2170704363655291, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0026009964674546487 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.03965743348041025, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0014470240504900656 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.09447416461397456, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.003039878935146692 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.05426109744027298, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0017575127883179419 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.12165923066163074, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.001738521026475504 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.27815207821679444, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0036330563038200624 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.1645988844863308, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0020230326221713562 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.12592899489421042, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0018469201981654569 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.28818031436396413, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.00395186316132106 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.17049497991799512, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.002199479559253552 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 2.286764120955454, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.10487671432151699 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_1.json b/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..2d43208be5f03f108f72c972adf49d20c62bfb49 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.13106642326274354, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0019193860865765633 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.3234658508585958, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0043661264753635486 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.18432241198153712, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0025753000481448644 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.02822515438220016, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0010434086746188042 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.07162285378667402, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.002714817358725964 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.04002835284879521, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0014781883740027449 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.10003470838188194, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0014061413510224204 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.24915600486179032, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.003347425480308083 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.14101261823875597, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0019068168040762914 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.10480543277054401, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0015545955460983404 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.2607965323547307, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.003724571304451384 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.14771150001250982, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.00211511203484802 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.601855503412538, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.07258015457329471 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_2.json b/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e186f951df9b5d0b713d4b46e05bba8e89a95f0d --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.13887280196688523, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.001956740885200385 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.34294827785468185, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004514777263625162 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.19535202200220275, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0026346536005903373 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.0320190408417148, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0011343925357226905 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.08177449666512925, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.003014469934741339 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.04546623699775454, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.001612092783689185 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.10508517578596618, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0014455778496582211 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.26185041154157507, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.003505120843276718 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.14811536601585223, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.00196180609857125 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.11141636053588261, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0015953318320649059 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.27772475736482033, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.003883573652450414 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.1571036308870091, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0021764703685117513 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.8380438639962695, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.10811928358813763 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_3.json b/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..3c3dda904785cfe8e70d593eec637e2f08a61194 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.13393465093211152, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.002178546476915688 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.3208349496275703, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.005017270430039093 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.1850973564810976, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0028685807268665573 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.030603540250172215, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0012053283031466103 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.07656078303232518, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.003042910587753952 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.04301170567438614, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0016643347985338668 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.10190253909758554, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.001662230819445601 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.2452262253929823, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0038665453486106385 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.14087163035130615, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.002174721918788832 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.10713139947740546, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.001810818001072249 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.2577398531586797, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.004224949592217397 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.14807083425641282, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0023776983157151816 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.8731170479468633, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.12212530111306877 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_4.json b/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..df60b77c4e75c07998304756237be2290a860225 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.04237213490568202, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.002644486260111252 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.07779789094117907, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004474695319161571 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.04957113221677667, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002776223278435835 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.00882760521824164, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0009353280430989058 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.017870091729374663, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0016736090478303408 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.010629103476078454, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0009487229867910699 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.032755367117331305, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0020851660139623363 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.059623992862197994, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0034445885527070548 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.03785078570554781, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.002102296416669563 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.03457603039331168, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.00217697156328438 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.06328733938197696, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.003695669480428956 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.040118179164564276, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.002244409483615815 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.8072921803610522, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.13481405520653453 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_5.json b/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..6c20da959d0513e1006078b64ee5954571ab2792 --- /dev/null +++ b/perplexity50/evaluation/generation/slim.perplexity50_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.002595925373248815, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0008354910877971658 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.001999825463947682, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.000634288246013444 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.0022401437562342683, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0007153938341072901 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.00027170862025409603, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.00014974174761724665 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.00019881893910750738, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.00011559008179568124 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.00022888298907867294, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.00012995777833085082 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.0016970776755226781, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0004959302048426671 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.001329703638978752, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0003765201577760315 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.0014744333692215336, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.00042158052355062295 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.0018375087990719742, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0005450511389629724 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.0014340716678750828, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0004083189310550863 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.0015935490543749545, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.000460457141424318 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 3.474750506260909e-38, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 2.4761840064885677e-32 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-perplexity/perplexity50/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/rankeval/perplexity50_0.json b/perplexity50/evaluation/rankeval/perplexity50_0.json new file mode 100644 index 0000000000000000000000000000000000000000..aa7d0d397e90c6b3affd4f6d66c47abbf8f6bc02 --- /dev/null +++ b/perplexity50/evaluation/rankeval/perplexity50_0.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.345, + "acc_stderr": 0.015039986742055237 + }, + "anli_r2": { + "acc": 0.338, + "acc_stderr": 0.014965960710224496 + }, + "anli_r3": { + "acc": 0.36666666666666664, + "acc_stderr": 0.013916893275819943 + }, + "cb": { + "acc": 0.4107142857142857, + "acc_stderr": 0.0663363415035954, + "f1": 0.23085585585585586 + }, + "copa": { + "acc": 0.8, + "acc_stderr": 0.040201512610368445 + }, + "hellaswag": { + "acc": 0.4962158932483569, + "acc_stderr": 0.004989638507409924, + "acc_norm": 0.6558454491137223, + "acc_norm_stderr": 0.004741208229092877 + }, + "rte": { + "acc": 0.47653429602888087, + "acc_stderr": 0.03006330041190266 + }, + "winogrande": { + "acc": 0.5935280189423836, + "acc_stderr": 0.01380444869775337 + }, + "storycloze_2016": { + "acc": 0.7338321753073223, + "acc_stderr": 0.010220104800551206 + }, + "boolq": { + "acc": 0.6018348623853211, + "acc_stderr": 0.008561755594317447 + }, + "arc_easy": { + "acc": 0.6355218855218855, + "acc_stderr": 0.009875729282482438, + "acc_norm": 0.5563973063973064, + "acc_norm_stderr": 0.010194308914521133 + }, + "arc_challenge": { + "acc": 0.29692832764505117, + "acc_stderr": 0.013352025976725223, + "acc_norm": 0.31569965870307165, + "acc_norm_stderr": 0.01358257109581529 + }, + "sciq": { + "acc": 0.846, + "acc_stderr": 0.011419913065098703, + "acc_norm": 0.775, + "acc_norm_stderr": 0.013211720158614751 + }, + "piqa": { + "acc": 0.7573449401523396, + "acc_stderr": 0.010002002569708698, + "acc_norm": 0.7633297062023939, + "acc_norm_stderr": 0.009916841655042809 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/rankeval/perplexity50_0_lm-eval_global_step80108_2023-05-13-09-53-07_0shots_backup.json b/perplexity50/evaluation/rankeval/perplexity50_0_lm-eval_global_step80108_2023-05-13-09-53-07_0shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..aa7d0d397e90c6b3affd4f6d66c47abbf8f6bc02 --- /dev/null +++ b/perplexity50/evaluation/rankeval/perplexity50_0_lm-eval_global_step80108_2023-05-13-09-53-07_0shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.345, + "acc_stderr": 0.015039986742055237 + }, + "anli_r2": { + "acc": 0.338, + "acc_stderr": 0.014965960710224496 + }, + "anli_r3": { + "acc": 0.36666666666666664, + "acc_stderr": 0.013916893275819943 + }, + "cb": { + "acc": 0.4107142857142857, + "acc_stderr": 0.0663363415035954, + "f1": 0.23085585585585586 + }, + "copa": { + "acc": 0.8, + "acc_stderr": 0.040201512610368445 + }, + "hellaswag": { + "acc": 0.4962158932483569, + "acc_stderr": 0.004989638507409924, + "acc_norm": 0.6558454491137223, + "acc_norm_stderr": 0.004741208229092877 + }, + "rte": { + "acc": 0.47653429602888087, + "acc_stderr": 0.03006330041190266 + }, + "winogrande": { + "acc": 0.5935280189423836, + "acc_stderr": 0.01380444869775337 + }, + "storycloze_2016": { + "acc": 0.7338321753073223, + "acc_stderr": 0.010220104800551206 + }, + "boolq": { + "acc": 0.6018348623853211, + "acc_stderr": 0.008561755594317447 + }, + "arc_easy": { + "acc": 0.6355218855218855, + "acc_stderr": 0.009875729282482438, + "acc_norm": 0.5563973063973064, + "acc_norm_stderr": 0.010194308914521133 + }, + "arc_challenge": { + "acc": 0.29692832764505117, + "acc_stderr": 0.013352025976725223, + "acc_norm": 0.31569965870307165, + "acc_norm_stderr": 0.01358257109581529 + }, + "sciq": { + "acc": 0.846, + "acc_stderr": 0.011419913065098703, + "acc_norm": 0.775, + "acc_norm_stderr": 0.013211720158614751 + }, + "piqa": { + "acc": 0.7573449401523396, + "acc_stderr": 0.010002002569708698, + "acc_norm": 0.7633297062023939, + "acc_norm_stderr": 0.009916841655042809 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/rankeval/perplexity50_1.json b/perplexity50/evaluation/rankeval/perplexity50_1.json new file mode 100644 index 0000000000000000000000000000000000000000..5e5275f14a0b38ab70feaf527d3403e278dec1b2 --- /dev/null +++ b/perplexity50/evaluation/rankeval/perplexity50_1.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.339, + "acc_stderr": 0.014976758771620347 + }, + "anli_r2": { + "acc": 0.313, + "acc_stderr": 0.014671272822977886 + }, + "anli_r3": { + "acc": 0.3425, + "acc_stderr": 0.013704669762934727 + }, + "cb": { + "acc": 0.5, + "acc_stderr": 0.06741998624632421, + "f1": 0.35220125786163514 + }, + "copa": { + "acc": 0.8, + "acc_stderr": 0.040201512610368445 + }, + "hellaswag": { + "acc": 0.49502091216889066, + "acc_stderr": 0.004989533998820352, + "acc_norm": 0.6565425214100776, + "acc_norm_stderr": 0.0047389206247244785 + }, + "rte": { + "acc": 0.5451263537906137, + "acc_stderr": 0.029973636495415252 + }, + "winogrande": { + "acc": 0.6053670086819258, + "acc_stderr": 0.013736915172371885 + }, + "storycloze_2016": { + "acc": 0.7354355959380011, + "acc_stderr": 0.010200400541714168 + }, + "boolq": { + "acc": 0.6131498470948012, + "acc_stderr": 0.008518188340844762 + }, + "arc_easy": { + "acc": 0.6439393939393939, + "acc_stderr": 0.009825454608416316, + "acc_norm": 0.6035353535353535, + "acc_norm_stderr": 0.010037412763064529 + }, + "arc_challenge": { + "acc": 0.3122866894197952, + "acc_stderr": 0.013542598541688065, + "acc_norm": 0.3412969283276451, + "acc_norm_stderr": 0.01385583128749772 + }, + "sciq": { + "acc": 0.909, + "acc_stderr": 0.009099549538400236, + "acc_norm": 0.89, + "acc_norm_stderr": 0.009899393819724442 + }, + "piqa": { + "acc": 0.766050054406964, + "acc_stderr": 0.009877236895137455, + "acc_norm": 0.7622415669205659, + "acc_norm_stderr": 0.009932525779525492 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/rankeval/perplexity50_1_lm-eval_global_step80108_2023-05-13-09-53-07_1shots_backup.json b/perplexity50/evaluation/rankeval/perplexity50_1_lm-eval_global_step80108_2023-05-13-09-53-07_1shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..5e5275f14a0b38ab70feaf527d3403e278dec1b2 --- /dev/null +++ b/perplexity50/evaluation/rankeval/perplexity50_1_lm-eval_global_step80108_2023-05-13-09-53-07_1shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.339, + "acc_stderr": 0.014976758771620347 + }, + "anli_r2": { + "acc": 0.313, + "acc_stderr": 0.014671272822977886 + }, + "anli_r3": { + "acc": 0.3425, + "acc_stderr": 0.013704669762934727 + }, + "cb": { + "acc": 0.5, + "acc_stderr": 0.06741998624632421, + "f1": 0.35220125786163514 + }, + "copa": { + "acc": 0.8, + "acc_stderr": 0.040201512610368445 + }, + "hellaswag": { + "acc": 0.49502091216889066, + "acc_stderr": 0.004989533998820352, + "acc_norm": 0.6565425214100776, + "acc_norm_stderr": 0.0047389206247244785 + }, + "rte": { + "acc": 0.5451263537906137, + "acc_stderr": 0.029973636495415252 + }, + "winogrande": { + "acc": 0.6053670086819258, + "acc_stderr": 0.013736915172371885 + }, + "storycloze_2016": { + "acc": 0.7354355959380011, + "acc_stderr": 0.010200400541714168 + }, + "boolq": { + "acc": 0.6131498470948012, + "acc_stderr": 0.008518188340844762 + }, + "arc_easy": { + "acc": 0.6439393939393939, + "acc_stderr": 0.009825454608416316, + "acc_norm": 0.6035353535353535, + "acc_norm_stderr": 0.010037412763064529 + }, + "arc_challenge": { + "acc": 0.3122866894197952, + "acc_stderr": 0.013542598541688065, + "acc_norm": 0.3412969283276451, + "acc_norm_stderr": 0.01385583128749772 + }, + "sciq": { + "acc": 0.909, + "acc_stderr": 0.009099549538400236, + "acc_norm": 0.89, + "acc_norm_stderr": 0.009899393819724442 + }, + "piqa": { + "acc": 0.766050054406964, + "acc_stderr": 0.009877236895137455, + "acc_norm": 0.7622415669205659, + "acc_norm_stderr": 0.009932525779525492 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/rankeval/perplexity50_2.json b/perplexity50/evaluation/rankeval/perplexity50_2.json new file mode 100644 index 0000000000000000000000000000000000000000..be2154b0dd08a01b70f1b917a587e2a220dd20d4 --- /dev/null +++ b/perplexity50/evaluation/rankeval/perplexity50_2.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.315, + "acc_stderr": 0.0146966319607925 + }, + "anli_r2": { + "acc": 0.335, + "acc_stderr": 0.014933117490932575 + }, + "anli_r3": { + "acc": 0.34, + "acc_stderr": 0.013680495725767789 + }, + "cb": { + "acc": 0.4107142857142857, + "acc_stderr": 0.0663363415035954, + "f1": 0.2854808590102708 + }, + "copa": { + "acc": 0.84, + "acc_stderr": 0.03684529491774709 + }, + "hellaswag": { + "acc": 0.4934276040629357, + "acc_stderr": 0.004989350311751647, + "acc_norm": 0.6552479585739892, + "acc_norm_stderr": 0.004743160034271143 + }, + "rte": { + "acc": 0.5018050541516246, + "acc_stderr": 0.030096267148976626 + }, + "winogrande": { + "acc": 0.6156274664561957, + "acc_stderr": 0.013671567600836192 + }, + "storycloze_2016": { + "acc": 0.7413148049171566, + "acc_stderr": 0.010126662138021712 + }, + "boolq": { + "acc": 0.6217125382262997, + "acc_stderr": 0.008482001133930994 + }, + "arc_easy": { + "acc": 0.6523569023569024, + "acc_stderr": 0.00977186884683091, + "acc_norm": 0.6266835016835017, + "acc_norm_stderr": 0.009925009142802893 + }, + "arc_challenge": { + "acc": 0.3293515358361775, + "acc_stderr": 0.013734057652635473, + "acc_norm": 0.3387372013651877, + "acc_norm_stderr": 0.01383056892797433 + }, + "sciq": { + "acc": 0.916, + "acc_stderr": 0.008776162089491132, + "acc_norm": 0.892, + "acc_norm_stderr": 0.009820001651345682 + }, + "piqa": { + "acc": 0.7627856365614799, + "acc_stderr": 0.00992469493358637, + "acc_norm": 0.7709466811751904, + "acc_norm_stderr": 0.009804509865175505 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/rankeval/perplexity50_2_lm-eval_global_step80108_2023-05-13-09-53-07_2shots_backup.json b/perplexity50/evaluation/rankeval/perplexity50_2_lm-eval_global_step80108_2023-05-13-09-53-07_2shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..be2154b0dd08a01b70f1b917a587e2a220dd20d4 --- /dev/null +++ b/perplexity50/evaluation/rankeval/perplexity50_2_lm-eval_global_step80108_2023-05-13-09-53-07_2shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.315, + "acc_stderr": 0.0146966319607925 + }, + "anli_r2": { + "acc": 0.335, + "acc_stderr": 0.014933117490932575 + }, + "anli_r3": { + "acc": 0.34, + "acc_stderr": 0.013680495725767789 + }, + "cb": { + "acc": 0.4107142857142857, + "acc_stderr": 0.0663363415035954, + "f1": 0.2854808590102708 + }, + "copa": { + "acc": 0.84, + "acc_stderr": 0.03684529491774709 + }, + "hellaswag": { + "acc": 0.4934276040629357, + "acc_stderr": 0.004989350311751647, + "acc_norm": 0.6552479585739892, + "acc_norm_stderr": 0.004743160034271143 + }, + "rte": { + "acc": 0.5018050541516246, + "acc_stderr": 0.030096267148976626 + }, + "winogrande": { + "acc": 0.6156274664561957, + "acc_stderr": 0.013671567600836192 + }, + "storycloze_2016": { + "acc": 0.7413148049171566, + "acc_stderr": 0.010126662138021712 + }, + "boolq": { + "acc": 0.6217125382262997, + "acc_stderr": 0.008482001133930994 + }, + "arc_easy": { + "acc": 0.6523569023569024, + "acc_stderr": 0.00977186884683091, + "acc_norm": 0.6266835016835017, + "acc_norm_stderr": 0.009925009142802893 + }, + "arc_challenge": { + "acc": 0.3293515358361775, + "acc_stderr": 0.013734057652635473, + "acc_norm": 0.3387372013651877, + "acc_norm_stderr": 0.01383056892797433 + }, + "sciq": { + "acc": 0.916, + "acc_stderr": 0.008776162089491132, + "acc_norm": 0.892, + "acc_norm_stderr": 0.009820001651345682 + }, + "piqa": { + "acc": 0.7627856365614799, + "acc_stderr": 0.00992469493358637, + "acc_norm": 0.7709466811751904, + "acc_norm_stderr": 0.009804509865175505 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/rankeval/perplexity50_3.json b/perplexity50/evaluation/rankeval/perplexity50_3.json new file mode 100644 index 0000000000000000000000000000000000000000..629ada20fb07e867e710008b8fb7d803f2559486 --- /dev/null +++ b/perplexity50/evaluation/rankeval/perplexity50_3.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.32, + "acc_stderr": 0.0147586523035749 + }, + "anli_r2": { + "acc": 0.335, + "acc_stderr": 0.014933117490932572 + }, + "anli_r3": { + "acc": 0.3641666666666667, + "acc_stderr": 0.013896714966807255 + }, + "cb": { + "acc": 0.44642857142857145, + "acc_stderr": 0.06703189227942398, + "f1": 0.3114930182599356 + }, + "copa": { + "acc": 0.86, + "acc_stderr": 0.03487350880197771 + }, + "hellaswag": { + "acc": 0.4963154750049791, + "acc_stderr": 0.004989645929811442, + "acc_norm": 0.6570404301931886, + "acc_norm_stderr": 0.004737279691036198 + }, + "rte": { + "acc": 0.5234657039711191, + "acc_stderr": 0.03006330041190266 + }, + "winogrande": { + "acc": 0.6045777426992897, + "acc_stderr": 0.013741678387545345 + }, + "storycloze_2016": { + "acc": 0.7466595403527525, + "acc_stderr": 0.010057563497401457 + }, + "boolq": { + "acc": 0.6275229357798165, + "acc_stderr": 0.008455846866956081 + }, + "arc_easy": { + "acc": 0.6531986531986532, + "acc_stderr": 0.009766326091716007, + "acc_norm": 0.6418350168350169, + "acc_norm_stderr": 0.009838331651451853 + }, + "arc_challenge": { + "acc": 0.3199658703071672, + "acc_stderr": 0.013631345807016196, + "acc_norm": 0.3438566552901024, + "acc_norm_stderr": 0.013880644570156217 + }, + "sciq": { + "acc": 0.916, + "acc_stderr": 0.008776162089491139, + "acc_norm": 0.91, + "acc_norm_stderr": 0.009054390204866442 + }, + "piqa": { + "acc": 0.7595212187159956, + "acc_stderr": 0.009971345364651076, + "acc_norm": 0.7671381936887922, + "acc_norm_stderr": 0.009861236071080751 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/rankeval/perplexity50_3_lm-eval_global_step80108_2023-05-13-09-53-07_3shots_backup.json b/perplexity50/evaluation/rankeval/perplexity50_3_lm-eval_global_step80108_2023-05-13-09-53-07_3shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..629ada20fb07e867e710008b8fb7d803f2559486 --- /dev/null +++ b/perplexity50/evaluation/rankeval/perplexity50_3_lm-eval_global_step80108_2023-05-13-09-53-07_3shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.32, + "acc_stderr": 0.0147586523035749 + }, + "anli_r2": { + "acc": 0.335, + "acc_stderr": 0.014933117490932572 + }, + "anli_r3": { + "acc": 0.3641666666666667, + "acc_stderr": 0.013896714966807255 + }, + "cb": { + "acc": 0.44642857142857145, + "acc_stderr": 0.06703189227942398, + "f1": 0.3114930182599356 + }, + "copa": { + "acc": 0.86, + "acc_stderr": 0.03487350880197771 + }, + "hellaswag": { + "acc": 0.4963154750049791, + "acc_stderr": 0.004989645929811442, + "acc_norm": 0.6570404301931886, + "acc_norm_stderr": 0.004737279691036198 + }, + "rte": { + "acc": 0.5234657039711191, + "acc_stderr": 0.03006330041190266 + }, + "winogrande": { + "acc": 0.6045777426992897, + "acc_stderr": 0.013741678387545345 + }, + "storycloze_2016": { + "acc": 0.7466595403527525, + "acc_stderr": 0.010057563497401457 + }, + "boolq": { + "acc": 0.6275229357798165, + "acc_stderr": 0.008455846866956081 + }, + "arc_easy": { + "acc": 0.6531986531986532, + "acc_stderr": 0.009766326091716007, + "acc_norm": 0.6418350168350169, + "acc_norm_stderr": 0.009838331651451853 + }, + "arc_challenge": { + "acc": 0.3199658703071672, + "acc_stderr": 0.013631345807016196, + "acc_norm": 0.3438566552901024, + "acc_norm_stderr": 0.013880644570156217 + }, + "sciq": { + "acc": 0.916, + "acc_stderr": 0.008776162089491139, + "acc_norm": 0.91, + "acc_norm_stderr": 0.009054390204866442 + }, + "piqa": { + "acc": 0.7595212187159956, + "acc_stderr": 0.009971345364651076, + "acc_norm": 0.7671381936887922, + "acc_norm_stderr": 0.009861236071080751 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/rankeval/perplexity50_4.json b/perplexity50/evaluation/rankeval/perplexity50_4.json new file mode 100644 index 0000000000000000000000000000000000000000..036624acd692d9a0034292af3096a6da639799df --- /dev/null +++ b/perplexity50/evaluation/rankeval/perplexity50_4.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.324, + "acc_stderr": 0.014806864733738863 + }, + "anli_r2": { + "acc": 0.348, + "acc_stderr": 0.01507060460376841 + }, + "anli_r3": { + "acc": 0.35083333333333333, + "acc_stderr": 0.013782212417178195 + }, + "cb": { + "acc": 0.5535714285714286, + "acc_stderr": 0.06703189227942395, + "f1": 0.39031339031339024 + }, + "copa": { + "acc": 0.84, + "acc_stderr": 0.0368452949177471 + }, + "hellaswag": { + "acc": 0.4969129655447122, + "acc_stderr": 0.004989686307484557, + "acc_norm": 0.6625174268074089, + "acc_norm_stderr": 0.004718846448021788 + }, + "rte": { + "acc": 0.49458483754512633, + "acc_stderr": 0.030094698123239966 + }, + "winogrande": { + "acc": 0.6298342541436464, + "acc_stderr": 0.013570454689603911 + }, + "storycloze_2016": { + "acc": 0.7471940138963121, + "acc_stderr": 0.010050543909878586 + }, + "boolq": { + "acc": 0.618960244648318, + "acc_stderr": 0.008493937524439337 + }, + "arc_easy": { + "acc": 0.664983164983165, + "acc_stderr": 0.009685160765932357, + "acc_norm": 0.6506734006734006, + "acc_norm_stderr": 0.009782853449399293 + }, + "arc_challenge": { + "acc": 0.3293515358361775, + "acc_stderr": 0.013734057652635473, + "acc_norm": 0.3430034129692833, + "acc_norm_stderr": 0.013872423223718169 + }, + "sciq": { + "acc": 0.924, + "acc_stderr": 0.008384169266796387, + "acc_norm": 0.918, + "acc_norm_stderr": 0.008680515615523745 + }, + "piqa": { + "acc": 0.7627856365614799, + "acc_stderr": 0.00992469493358637, + "acc_norm": 0.7622415669205659, + "acc_norm_stderr": 0.009932525779525492 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/rankeval/perplexity50_4_lm-eval_global_step80108_2023-05-13-09-53-07_4shots_backup.json b/perplexity50/evaluation/rankeval/perplexity50_4_lm-eval_global_step80108_2023-05-13-09-53-07_4shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..036624acd692d9a0034292af3096a6da639799df --- /dev/null +++ b/perplexity50/evaluation/rankeval/perplexity50_4_lm-eval_global_step80108_2023-05-13-09-53-07_4shots_backup.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.324, + "acc_stderr": 0.014806864733738863 + }, + "anli_r2": { + "acc": 0.348, + "acc_stderr": 0.01507060460376841 + }, + "anli_r3": { + "acc": 0.35083333333333333, + "acc_stderr": 0.013782212417178195 + }, + "cb": { + "acc": 0.5535714285714286, + "acc_stderr": 0.06703189227942395, + "f1": 0.39031339031339024 + }, + "copa": { + "acc": 0.84, + "acc_stderr": 0.0368452949177471 + }, + "hellaswag": { + "acc": 0.4969129655447122, + "acc_stderr": 0.004989686307484557, + "acc_norm": 0.6625174268074089, + "acc_norm_stderr": 0.004718846448021788 + }, + "rte": { + "acc": 0.49458483754512633, + "acc_stderr": 0.030094698123239966 + }, + "winogrande": { + "acc": 0.6298342541436464, + "acc_stderr": 0.013570454689603911 + }, + "storycloze_2016": { + "acc": 0.7471940138963121, + "acc_stderr": 0.010050543909878586 + }, + "boolq": { + "acc": 0.618960244648318, + "acc_stderr": 0.008493937524439337 + }, + "arc_easy": { + "acc": 0.664983164983165, + "acc_stderr": 0.009685160765932357, + "acc_norm": 0.6506734006734006, + "acc_norm_stderr": 0.009782853449399293 + }, + "arc_challenge": { + "acc": 0.3293515358361775, + "acc_stderr": 0.013734057652635473, + "acc_norm": 0.3430034129692833, + "acc_norm_stderr": 0.013872423223718169 + }, + "sciq": { + "acc": 0.924, + "acc_stderr": 0.008384169266796387, + "acc_norm": 0.918, + "acc_norm_stderr": 0.008680515615523745 + }, + "piqa": { + "acc": 0.7627856365614799, + "acc_stderr": 0.00992469493358637, + "acc_norm": 0.7622415669205659, + "acc_norm_stderr": 0.009932525779525492 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/rankeval/perplexity50_5.json b/perplexity50/evaluation/rankeval/perplexity50_5.json new file mode 100644 index 0000000000000000000000000000000000000000..a57390fa41a2b8076ca3c7b767236f5f806dff27 --- /dev/null +++ b/perplexity50/evaluation/rankeval/perplexity50_5.json @@ -0,0 +1,59 @@ +{ + "results": { + "anli_r1": { + "acc": 0.327, + "acc_stderr": 0.014842213153411242 + }, + "anli_r2": { + "acc": 0.329, + "acc_stderr": 0.014865395385928359 + }, + "anli_r3": { + "acc": 0.35083333333333333, + "acc_stderr": 0.013782212417178193 + }, + "cb": { + "acc": 0.5714285714285714, + "acc_stderr": 0.06672848092813058, + "f1": 0.3985557884928954 + }, + "copa": { + "acc": 0.84, + "acc_stderr": 0.03684529491774709 + }, + "hellaswag": { + "acc": 0.49422425811591314, + "acc_stderr": 0.004989448490164432, + "acc_norm": 0.6600278828918542, + "acc_norm_stderr": 0.004727312448892851 + }, + "rte": { + "acc": 0.5234657039711191, + "acc_stderr": 0.030063300411902652 + }, + "winogrande": { + "acc": 0.6203630623520127, + "acc_stderr": 0.013639245403711156 + }, + "storycloze_2016": { + "acc": 0.7487974345269909, + "acc_stderr": 0.010029364281941636 + }, + "boolq": { + "acc": 0.6204892966360857, + "acc_stderr": 0.00848734197575683 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1 + } +} \ No newline at end of file diff --git a/perplexity50/evaluation/rankeval/perplexity50_5_lm-eval_global_step80108_2023-05-13-09-53-07_5shots_backup.json b/perplexity50/evaluation/rankeval/perplexity50_5_lm-eval_global_step80108_2023-05-13-09-53-07_5shots_backup.json new file mode 100644 index 0000000000000000000000000000000000000000..a57390fa41a2b8076ca3c7b767236f5f806dff27 --- /dev/null +++ b/perplexity50/evaluation/rankeval/perplexity50_5_lm-eval_global_step80108_2023-05-13-09-53-07_5shots_backup.json @@ -0,0 +1,59 @@ +{ + "results": { + "anli_r1": { + "acc": 0.327, + "acc_stderr": 0.014842213153411242 + }, + "anli_r2": { + "acc": 0.329, + "acc_stderr": 0.014865395385928359 + }, + "anli_r3": { + "acc": 0.35083333333333333, + "acc_stderr": 0.013782212417178193 + }, + "cb": { + "acc": 0.5714285714285714, + "acc_stderr": 0.06672848092813058, + "f1": 0.3985557884928954 + }, + "copa": { + "acc": 0.84, + "acc_stderr": 0.03684529491774709 + }, + "hellaswag": { + "acc": 0.49422425811591314, + "acc_stderr": 0.004989448490164432, + "acc_norm": 0.6600278828918542, + "acc_norm_stderr": 0.004727312448892851 + }, + "rte": { + "acc": 0.5234657039711191, + "acc_stderr": 0.030063300411902652 + }, + "winogrande": { + "acc": 0.6203630623520127, + "acc_stderr": 0.013639245403711156 + }, + "storycloze_2016": { + "acc": 0.7487974345269909, + "acc_stderr": 0.010029364281941636 + }, + "boolq": { + "acc": 0.6204892966360857, + "acc_stderr": 0.00848734197575683 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1 + } +} \ No newline at end of file diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0fdafbdb417185f6fb4b5da543ea87b50d4e90c --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d468e35f2d19c468d262ac502fef5723fe813d308ba8e222555238f4ac7c4d77 +size 199058647 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0fd3539531acc4bb273c083b866a8e2681cfa44b --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c036444dc1574264cd6700e9fd9dc19e2724229086a893c95423fb4ab856eb0a +size 199058647 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08d8633678f6420be1b337cb27e0c15c9525a3cc --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:714f925e6db69daa8c9a640671c8580bcf951b6c19aee781df5790cc24313ff5 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d46ce946b8d57f68b2a3d5cdf5c9fec9e676ecc3 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b72de7670e789a2640bd5881a9e8f1ef10f582b7f3f02d2808c0a93b8c04a9cf +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b199805bae5dbc030618e7ad46d07cb681106fce --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37c7840fe09f9ae5619cf347c44306f09ab252c2a1ea6b5754d01ca39b247fd7 +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c425784d7620a23f6c26f36e7620fc5217272c59 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e1f62c05720bfc869b6385a0cdf345fad2faf851aa26a0538b396b3c9a1c969 +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b23eaa6b4073bed7157aada3a6e1466410ff0c11 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a96314dcca295a5b8f756de6f284f1f021d42a9d9b60f94fe0c7cc44ef0d2078 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0eb708a87f202ee36c57372a6f1e65cdb6701b53 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47a4d1fabeb31f4c8ff68c7301f8ac2b2114d63f50b3d6d6e459ffd4c7274876 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9648533a6e56b23d02465d50ba95e7929801295 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15d87ea6df4ccc1f908cb7f0310bd4d07bca7beb73db69536f70acc3f6e523c2 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ed0187d2b1c8991ee6e76ac6cb204743141b4c5 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f12795d669be8ab2843e23cb946bf60b764348081f64ff290567887f5b44a89 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9168c23d8f8502b6a0d5ea988707c19eb72f473c --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c000a06f56530cbb5e7e7fcd64814ecbdac74575d2b061bedcf4d22f0a2d4483 +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eae7ebee61552c14c1a74de94cc8391559811b84 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b180245cd13bfca1993063beeb7069b0f25c909d50e0a61ca67067119cc23340 +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8908074b1dc75394110bed58386a24ad36bbb7b0 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:111eb2de2ab20ab3a73819b16ecd649458764e022c9fe3b05d70644ad7c4be94 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd27f194933717b6000dc06e2adc76ec72b3f498 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd83c1cfe780e56d46f69b4cf40d8254bd1af24d4c0ca6f30fa674d5a46773f8 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..626aed6a3653a84a9ee724f41c14537a166382fe --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ab98629c010f228ac35b7412b35d2500704ff8bc8aff3cf45cfd8f69f3e8ac9 +size 199058669 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..068b06170b26dd7ab929a477d8620de735310fd8 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a5bdbe09beafb2dc591b1abb19665c66d861d00d520cda274032cd1f525ad24 +size 199058669 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..64184d446913ec357ca5fd0dbfbd27787d27c072 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56546ad2640e5f59d28fbcf053737200bdcd0df0a306828419b536a29cbbb237 +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e2f0b4077efd4072d5d0d90e1c5f6eba7949b25 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93ed5253f377813471afd4c102de9e9888b8708b44b92ad3371015c437e4dd84 +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9652887c92bd2aa931baea5eabac152b378b4c2 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6db2ee7f503547206cd69e2de172de425726db8a565d3c7c99d4d65da1d34322 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b14a73af38ba20b89ab7acdb38c200d5b3d1eb03 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c829c86bc0683a4b61312143f5633306f4a46f828b1f413059416f418e0f4f3 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63958804f5bbf1c57c7287d0e587b461332fc43f --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e93b8e8df08098069374178afac180af2472d7cf62370c522c4ba523e97ca7d +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b65f419ee857627d898a9136d6244a963ba8184 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3039b36cd1903616b320f3c6baf201d19c0de5704fb98e044cd2de589c1fe612 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..efa3d2e1372589542c8157ef6fba8ee9274f5feb --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c399342c2b413f0e81d1b06a2e0d05e94fab146ac855ea37bab1fb66eb3af22f +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46e154c21462bf1bfa28c7c1cce5cae794192a0f --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82ddbefffb30bebf3f09a300fa818cb46f93e59d02e6b5c6001b693e6bb3b110 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f7b27c353bc706dcd5a28ca10a0a9ec7595c574 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c48ae8d4705d349659a9e3db964faa7878753ee67489ec43bf23fbd50306e05f +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..121e40901827ed8c9d6c105172d0cbd23f6d6202 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6d137e2e34006ee3c2f16ffb048a96ba9ad14c01d3c967bc6dafbe2bce95100 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0109f0fbfb110885bb1f9b4c1331e7a7bb57cf9c --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa27e3babdc1051a4582e9b2cd2ca620e198aa814167fe121ad33e0d57eda750 +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7d23060b5469bdc6f1c4d5bc959ac8b5fc0732a --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e688d8fa367fe2f567cd81aba94fca6c38865cedc1f7de6161d7f36aa67008c7 +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e989b043cf8661fecb8d1a1688866c5deb55bfc5 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0938a1503217a474f1da736675962966955fe304aa8fb1bda5d187f04e4ca17 +size 199058605 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9da7e96d0c4c7a51e6b6dd59c13dc828165b699e --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68ce4de10dc7adac7c6d47b065c387f2663db60cc369e2f09e2c8fb9b12e52ad +size 199058605 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5cca29efa6d0cd2bbe58e3646fa44b7de1c4eaf --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0817ba11b91ef679d382d3e9ccb9da7ad662a9c0ba6793d5aa1bb828cb4962fe +size 199058669 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..28e0d47f77334964972a9766c2208062c007130a --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee9afc3d63710555bcd56ce97c263b7798f8ab68b6b739bb6f7296305a56943 +size 199058669 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a047fd47ab4a243027090986406a05c3744622b8 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b8650187ed1f23d429bcc7d85170549bd080d82e27c4dbdf4b2c6db10b8ccd3 +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f142ec0fb503c3884ea0831835d1301f1258cdf3 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0337c36853c1817249437590191cc76cdc303a58f861e15ddb9fd195ee1c41d +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff72a48653db73e7fa4f78d6db435052223a0ac6 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9487ddb2dc783483eb89ce3dc5ff836a000439957a764d44edbfaee546b8298d +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0bdafe8c85c97764be6b6a4bb4f8a3785ebc7dcb --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4d167bbe0861e93efc27eccabaf30eb459f6b4e5e96fef47f65904c8bdb27c7 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c3483c9880f5963c47e2eadf6b8dd20d051d024 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f79d7d284dd2c16eea2fdd52222cad753dea834a80137147a238310a585c251e +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff929756a1a09bbb47e0bab2af7fd81010eb2293 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8fde67134613c08d82ac0369b38a3ec7e84ec5acad5e30699264088c2f2f7ac +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6ad5c8adc8913c5e966b977258452e5a55db8c2 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba9eea7efbad216ea270077eda5cb6672f55803cbfb9868fb8604ec3837eb8a2 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..279b1ad803994d3b4c3daeeea7a912cedab8d788 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baf2c92301fe5efdc84c6e10b8e0d07adae90d1fea9911f77670746b42f52fd9 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46ebb0acd238a1bda923ec58956a58817f791bd5 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4045ab122f57449060842940769d61c60b72873c4d5f9e6de010e25409bec6a +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2e3f49bd4da4fb910ebe2fe1a00e8356aefb9a8 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74625b4593e15e8c4ca4b409fdb8648b2f378d4ac176b0837e1380047e5aad47 +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6f2b049870a358167ae371a2f53acb357aa069d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e399d5ac16fcced2e6bbde03e8d2f10dafc91a24a1e92c2afad0e61e00b69c6 +size 199058669 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e35543c5e026de2a277999fafb67d72900b53ea4 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66b6820f767fd8d52fc9596244cb1a1078b569483077d9240b6a5f3b8dcd5dec +size 199058669 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86b4538967a2a1c49d46ca08ae0d7ebbeecf3c6f --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:589454b3a8b921719ef84b02dd7e292fcff230a7067008a4edbb99d17bee60ff +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db94a60f96dd9c890ac43e7f018c44b3b2a3f168 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43e26d21e74d21f311fb8852c076bdd470561f3ea0137b8db0fcd6dce131c3a4 +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a8db12cf969f9c1c04e8f108e8ebd16ffaa32b2 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0567ad1800d1bb7ab9e716293adc63e68bd090b0129d03f121d43634d32076d6 +size 199058669 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..245a0edc87ffb9297bf4caf044936d37a1c875b4 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10358907ff89b8ebbf110258f1d7c67b949a9fd24bb9dff12eb5a658e91d8473 +size 199058669 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1ff84ab553153b5dd51fc5065aecfe043faac49 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f7ebed99d68b74c151478d49da23bb5e44eaa1fba1cd8859f102eb1e4d269fd +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..67ce1da3a81c522d162d0af1b21b06f5b78a48e1 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:991e997031f51653776340c896aa1196899811607c3324da10b7a1b528a4965e +size 199058797 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc9bb95abe485182b95e3cd13763d7a81979fc77 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3c9127916de6d6e00bb63f592358ff58e23a83ace42e85b25b9a432fbdb8960 +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb5452388861d43e2f98f8e8f4d5c3037c885463 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dd46783d927278ceacb3c4b880c93d90583da935e67ba1b1f62967449b1c8bf +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3f461978f6812f6eed56fce361643662de82677 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0edd6f25b678babd832978935e319d125703fe775fa26e35fde749bc1c459fdc +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b9edabd7d26a6ce5d7c39d89df0d5bbc7a224f4 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f63c1db672196fe3426f6ba258635675d00cf70f85a5dc06f2dfcc3def9b6ff +size 199058733 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c03ab72e13060f5953dddda8a46523a06c78067a --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c7158c0271c0b47d3f21bb68a189cc4798fca8fa47c8af98cd8871b90d963f3 +size 199058669 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd8ff3cb7f9c4b5f82c81499fca0b464728bc289 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b14f4613258e134ce108e8123331432aa9f78a7b16ff19a43dd773bf317d75d +size 199058669 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1db820749d3ac84e58edce0796c6f6c4898efd88 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f3946e665a18a5ea251bdc6785b5b357540578825081a36a7b23383a15a1112 +size 199058925 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fbc02e51a3f3191253a93deea02ae5eee219cf89 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c154763d9a126ac1d1ca3ec649ae0c022d8662bfb394b71e0e12d140be07d98 +size 199058925 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d72ada46e8891b669338f0c27f48bb09bf344cc2 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e845c4b90058b45c796b5a777be1be6b1388dbbc9a8c8b50d38225c6900b3a25 +size 199058605 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94d5404421628e9c6c01fc6f08b63516abc43618 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c53c0f412409ced1d9927c84c0a71a384c0f3b4cd4a79f2b6936b38f0c06959f +size 199058605 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df69646096daaca6ec7a5fe7d1b256da158825b5 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1c96425c8770433e96fd2a3a2a46f1ce1c0052daa4670d6f72c8c1619b3ab64 +size 199058605 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..28b8f678d062e5023fddf4d42b9b4742fd3320b4 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:debf22d3c32345d86410f90653e7973e7ec49db8144293f34aad12686ef3a751 +size 199058605 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d32357570bb094d73cddd9ff8d4d0d89bb54237d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e96ff37f4bf58f58ac2663d86742a1e041e15ee428c4329d8e022b411c4c978 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c88d679bd5998773d2fcfa6ffde7cec5866f427 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6f62ca5470c9425487da3237eece7e2a4381d2521317080661a06bdbe878825 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..932da95202fce82921582b981d44f03e9bca5d57 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:138d19f1d8174bbf0caeb202b7602b890af9806a36e1413485afd69896ceb5be +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3eaa7e3016e68fa23ecc2f0f2116f665d20ef0bb --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8d77c83cfaddb1a41db0dd0cfe1d706f7cca1e9cf1ef7d7dd6ec5a371f93ab8 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5f5b4b463be65f3a56732fc1640280f4b4433db --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:761d4a1216c762def186a6465ae38472fcd0a5ede9f072b3adbd86223a7f0ced +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..317d73c27175b3b100bf511e948184416f210d4e --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19d89f943fc145ed4f837a28161363d4d8966c7a642853759dd9d1e7b1176632 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..294cff1b100584c275085087621d46cc5906dff2 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4397dd812d7f96d4fa4e37e4dfb41fd1162f2d37d38066a25ad16f4c946c8b44 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..223f0f4d5cd5130ee0398df80d6ed3053b0486ca --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5b1ec3c1b489b4bef1372594e86df75346cdc2f7e0de0632648ed0367ce9373 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fcab1b540cb992ef497b092e9150b64dc46c98c --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2325f37762eaade9ca77439bd4afef25a3e2caf12ed1a81e2483ed19524c579d +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..992bd1aabf798cbddb1dc8dbe5752b57eab90b89 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:324d9674b2a95d20154d54bd74f2fd48bdf2d058bcc0d100d19c430458b4e504 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af916a4af6ac0e608fa95a942e5ac3d246e1c54f --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:813d671f3108c0a81f801ec5323d62a194945a977926011559f50420cb08bf00 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6b7e54c7fde321593d044bc27e89639dd271df1 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6ea20d98c0d6fc1f23e5c241f366c2019c240e9cc704d4c61aa600da8e7ea3c +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b12b69b6b69ccaacf466a79705c6c19d0fa8286 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8c7ecd616f3fb1a5aff22d844ef8a168911dcc765ec7f67518b4fd60ec6205a +size 199058978 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c30e6b479b61df8ca1fbab8e669f4b82859f7eb2 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:789fe984811e0896b8f215f97cb1d53ff85d75e97efedb29e28287218a57569f +size 199058978 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32d0a64bbc102e0cf4ca538b9754217c2d300ffb --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4194ed96ffd8e70da129c116cbc1b188b7ed43f92ef935309172fd15aa177544 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f91b7cdae35ae87edc1421774fb6fe12ca694d6d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:722adb3ad59859b8f7d96cd38d248968f264b7243038abf3d163234e8c0d0ce3 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b32f14d9cc578fd5d9bd712166270c7826209c91 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a5340c58526aed00cd59de591df08f50f949f6e8cfe40c7e5e7bdaa3a69f88f +size 199058647 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb5aca152ddd2cfe3461e70764463dab556f53f8 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ccb1bf2118a6b0ba44887dbd47097f62bc1e0cc177785ced05d10d994a75a77 +size 199058647 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6465e5fdf03d097bc6fad14f9f02e8c0dbef6fa4 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eb12a8d6eeacda04f10d5b2ac9bba13cd8506da806d361b4053823462540343 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08123ea212e581d5b4dca168aef0b2e03aa19725 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77becbe9c7917b1980dd920a3f99b7bf2fd455f76a7d5221deaf838051574bce +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9931782c9ceef439d037869a95ed49542b07ab24 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2380d906e47aad4fc1bbc3d9552828843bb43771c60bf38141d2b72735f6711 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b38f2bbf54e71f0c9cd7401d75b36b8e61e842a --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:872f312c037528c2287a842fd3ab8db0b627ea2fc10fb65d54c798589a75e398 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dfad647cae93a34dca648d767b330030b0532f74 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32896707d17a1b3378ef45087f376f92f865efa0316c45c73b5592f7a1bf9302 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5089895c3fc9ce129217d14bf7ac1657b6491f2 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94de55adc2134c3b3e263fcf1c280554ded057f5e645c147b07de27a721805c7 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..95ba208541a45bb8804a207dbf771a54f9982b19 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a06cc8099cb0582549452644a38b0f863b0c3e7f8580d55c831d93f3b8f98bd +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8423b5177c6fa6174ada26f84132d4d95c4dfb42 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:035e8083b66125b9005c063fab9fc59f453a76543bc530af3831f6e1542cac55 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2af5cb68d966ac317265c5ccf3aabeeee2efc85b --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7d86eb0cc13f601cca4ef30d586d41c1ffc81e69d3f45e891d83d79972678ab +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21b534d76feee19605f06eb8a46dea5d1249f9b9 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:708d418018b0dd55d4e6e1603009c9d096ec49dc2e255f8557a84f71644e7426 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..09b831b75fb19cf102113970557e6e0bc924fa3a --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:098b1f08e47a3ec4ba13fdd164ecc72e0fc512d7e5d44840a20d3391535b5447 +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8dcf141d41504729c671bba1327da8bc4c13d1d6 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1c597078ed5652663aa59ab9f42b3644f2408330a75dbeb143c49824a587451 +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..133c1ca3fd7c277746b6b854a4411a2044d7ac7e --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cbc3346f29520dc1eec079ba9f95d3ed59311bd35ae7c1fbd7e2d644f024a38 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1873e21b8ba769d6995359d37c77ac6faeda9ce2 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2be1e8f57c2d16021aae42fcf17d1333ca4ca8d294418833abea9ed1851e785a +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..27d0bc64a1e8edcb8034461ed00d9a6c6e9e0ac1 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7cdf24a99373e993bbe30345cf988a93e2fc3c538082b2e6fb5fce0cbfa8718 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35cdbdf0cc7f3f99bedfe8ff1c73cc7e12ee80a9 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b109f1f36cb0f59f8d1a78ee6ea5ec9e185bf02f5b8b00d593b4e33c7705ffb4 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..117593c3401c6374a2d56d416e87a5c7d870f61c --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f025470e7b8d128f2d367d61875d3d1e6490841072759f3f91ed2e285c27873f +size 199058594 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86783b57a8a0cd1645fc34ee31af132698c25bdb --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7d39a7887b53bfabb82c3cd28eb3bad4725139a9455a0838a241e6fa656a3ac +size 199058594 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eda7366e82542ff35d4ba7dd6a007820252cfa6c --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a092abdbeea4d6ced0dd75f1090a3b6bda556eefe5c11e978fea9ae90b481ea7 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a76c6f5d96e5595cc6c93563be5ded93259980e --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef523882620c1b0fd5c7bef42adfd797ecd8338f9a11ce8ce05d42f35d5d9ed6 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ca7a8a2c37f190698684365f02b9be3c3aad0c3 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab6a3670e6ca982c364fb4355634036a8ac877462773f8ae0fcb826ec541b2fa +size 199058711 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4f0d91eacfcea4d7185450327622890984acb45 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b664245e444825b0291047fdeccfc098f1f795a8c4e3c7746bb63c85856bf62 +size 199058711 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a58475e5089f6a4aa921fa0a201911d7df7862bb --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca752a033061a55de1eacf882d506133cdf295715da509501d48e56c5edce13e +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f3fb5b92f09b84186e98836e716c8706111fb6c6 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:febef3ce45bb2ca4741f942ef770783598976e637ae4fbe2a129fa514b663df1 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d46740817400bd4c14563f561ef467b06f68465d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:909529061252ed6487014e6d41f680b9f1030c313388688f07fe0a5edc2d94b1 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b65e489a7647151a847888211d489c1e8cdd38b3 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2671fcf88ef318ed48511e3d5fc62cc61062a26f6b9cdbddf4d9b4ec99a20d27 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d7910a0c8ef21733bd7d15611905a12e9b50558 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3380311180321ce01ec2b60463c9a8a2dd823a1cadbb5fc34fd07645b480aacd +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59540ca98562f040c7d9204f1d730bd08a56df9a --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08ed47316de7a57022ab49528c83a5d5103ae154c7e19e81b2c53bbb2cb5f4d3 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3fa8fe83a7700058f6e86e6c4139c508def18614 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8db0c185e37e0ecda9b8d27564c209f6081bd4ea5ead70c63a52a5dc1043e20 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..295eb69b52d9212a32a86ce816db5ceb856e6a8d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a8e0df3429d7b26a62a352e6959cc85e111183cca537e5cc7ea227c3c09f5da +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f33245dd31dbf6fd3703549c4f3dcf83b80f41a --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3a0db99c6de3d083ad457bfe6acb580fad196ffad95ee9f6398f3940cdd9298 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37a0e8ab1c9f66f7064254631a778c5438eb09fa --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f00e5145c6d862a53a4d4442add102e95f57e9b9d46570379160bbf7c02bffcc +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a21ad94aee054ffa5561d26673b9d16993b3651 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07ea6ef8f8bd3241d4744cc4f91816ca404808e5764b1f65307e08f495aa7cd9 +size 199058594 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd1f62e829131964ce8efd04a742a594348c7734 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dc19d826f2221599737ecc744ffb6504106c7897d895eccc51c31f3abf2a537 +size 199058594 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a12634d11582fa5af6f5bad8a25ae0c8658d821b --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de21b35e303bcd96011312e13cfd4c3038f89f95ebdb780b1d0ea96f25b0b1c2 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e59039136207f9d4fa3df1955b836e5145acca83 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d3d48a73937731aa550056a02fef6cc4a10bfe53bbb970e6d18a740870722f5 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe581746344d54ba82e53d97adbb9fef5f5410c7 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:909efaa7658486328fe9f2dce9eec1889f65d9bfeca734acb318fd095ffa55dc +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a914216aa9010f70cb5f9acc764c99f204aebc3 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62fa5605266517deef193ccb3cacf280f6ee60f5b62fea1dfe3e08b6b8b4c1f4 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ec41043d9cd9c77471b6a31c43161132039bd94 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eec99383859220a265b9ce2f85f9330da83c8e0a1b81b7e95310047ebb9dfc7e +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf00b961733870bf2732284521b12803dfb53c2a --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58f3895b98c25e7d42db46e74045a72795829f3cc1c3cc1eaeddf611c2b27a76 +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7641455eb027ac62ab9c7c1b0bb7a97329b9c14 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dafdda2c91527ed27ca2b6607e9b2dea166fc8864d21600674ddc11f6c6cf565 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc0f5cf8f00428cf78b04ea8ebd4b0cbc560f97d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9179627f08e2eeaffbf7112719eead893f090038484bf80fcf83c157133f16b +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..236dc0e80fb7adc90821fbb48f1bd83321f09d25 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17e8ecf9bf478999bd94bc03b4316461cd239c35abaca089feaba9d2ee24b5ac +size 199058647 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b80d10e74684f8013af844b3798f558e83d2518 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b719daa8c0a4dd0c1df113b7fdd1686ed053bac498c11ee8fbf5306da97f48a5 +size 199058647 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7bc3d9657dfc35ca057faad272bedac9cfd7b173 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b52b49572479e67a28703cc9c54bb3c8b431d66c6e8717558905fd0e6d91ca56 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d3a46e06e76bc5dce9783487c5996aee6c76dbf --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98418aa52d48e6a961d92a5c1b73d10cafef2352f3b80e0a76b580a88444cbca +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ace25d5ddb2f4d8018569570818af1ba34144c63 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f93ad0a4c7351bc38a6e9ada78a3463c0d24f62ac6a94daa3cb2d1336d483e46 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5633072ed3da96fd60fb8c19a5bda49fe083e7ed --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2b82b5d3d2ed3187628fa610352d94c942d9263cc70e7c012d335f206284b55 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..942a7aaf56aebb8d69a31d0b60162a5a8e7072bb --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb4ba000cde57328c8502b4f6dbf97701729b0dba7c685878564d8c1beee8540 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9307938b50fc6b6fbb73c1cf58c3631bc778974 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f9906096f05b5f3738ac77721399a653eefff3658d7c4829175e510f04aa02b +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c3c25ddb1a775126160e67478b276470a446360 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efa12f804a3314db01d578f2d7d2e38109ed987ac4e3eec5cb8cb7fb59d15ab3 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca84c60e686d6c07e13c82a9224061b6a29d3761 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d7325741e91894eb95ff767ac5485cf6b64fc87e38ca10359539d63f9b307bc +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea50913205128dd5aa97db8e20fd888b5ee5e94d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19105976921412eabc3d57daad8f631e883ab8727e3d8390676e8999015a81ec +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1c66357975713513b60c3c24d73e7f2917791de --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d802ab246f1d17ab61dc3b67fb4eb5f99228abdfe538c662635f0b84f19c798 +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fe888f18d90fc2f98e1a94722f6f1fc463417d5 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ceb0e39885a4343918aa66c3ef7ed046e1989f4118028ae732439d31c7207900 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f0eafa2bc155722db6431477022f7a831dca950 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64559ac8833d71d122b8aaebe7c118d64ef24fb5b0da563b9445208acfde69ce +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..defebc56858dc8e8c3b1215a15b6bda022af4f6f --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0add89a217881ed9ea1d3f5e2f1d2a99feb4d2908bb8bd0bf454186be67abc5e +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..04b9bd697a15d38975551ec9f76ca8d8b8f03217 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b41b7916ac24607ea45e5e4df97ccb14bd161ad9962b62fad0de842f493da461 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b30e6e546425d4f0944b14b7a2014b82dd0776f --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6f2bef34b192c729df008ea8831e94ce1c2d76939ac6ff0eafc5c04c8be89ab +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88bcf7106af636fd68733acf43c680252c07ec82 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:895541bf154ad99d432bf7f7517a26fbb43da69f8848bd43ab600bf3aa41c4e1 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7adf723f68954635b144f307936887667b90a66d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bbaa12b817b2be808ee07a84455f56e59ca145cc003219b3c56d77f4875690d +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30593ca7ec557852995c0ebe497ddab41fc52a9c --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8696a5cc466f409a33072de6f3d314b06f42d196de14effff20ce34446255984 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6965fd314887c221e12c38a4f75d01950884feb6 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d57b221ef01e765362283d7684676fd65301b36fe7378ab2e2610f7fb260b06d +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d7759daf78f934ffbf32d77f7d8994ea22c3f4f --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3346b4aaa1418d55bea2fd081e5cde9895c60f04c43a46a69c0d6b660b9aa28 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f22f53fe479028e8daee82f24da8c7e196be3382 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74cf857b624153febf149c04448ffc2908fd0d6ebbe0a59a3b4b97a83106cc06 +size 199058775 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29662dd042e42b9b9fbe6f6e3805c5c6c38fa285 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e479ba534e8fb3b79cd3ce9aaaa353b5c8963897da08ba028c62933227720293 +size 199058775 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22ccbfcaf1f0ed3defc8a8e14dbfb8159640777f --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d00a68968feea807ded7ea75f6a426599a84a57a163e1225e27176e34e0321a +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5410143a19e4acf21886b42381df558147aa3509 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:316ab3ae3a4f10f3b5a227ee9322b7a20b626729d9c5db5be99f73b9f84808bd +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec62ac3d1faad133c8163ff9026ee8a96c6d0f9f --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc5f809fbbafe05aba3e20d3e99596fc735cc307a1c6ff1bbf00a54674d72d50 +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e36e8b8652c3c1b08d9b372f25bcdf5e839e908c --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4da2908d5d95341e85bb54572b10c97bc0bc0b0647a01329279d3305ec380456 +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9603c990ebdcdb93d4a1cb8f60537afb66aa944 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c5c33995bb3dddffdb09c6ba6aa9609d1b4a7e43c97024fd64a27201c4057f1 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e5a98cd7633f7351df11b762b6291ab8cd47dd61 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6dfc9b621b179af4ce8863545388b0ca30c45c8265b70b760bbbe9bf7efca46 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db3ebc2be58c56a149b7624c444616d35fb1444e --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8f8e6f75421ffa9f9316b137d2968bc7e1ec5d07b4ff9888fdf616499d4b51c +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89f78ba9ca0b68a4728f51f7f12890aa1effe03a --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5af2503fd68b634327f27ccf9bda3a589d56590dedc9108acc265b5c185bc47 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ef2bc9109300a3fb88977d0ec39e15ee2a5d86d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c41eed3712c4d987f7cdc574fb162f8b64588461aa446f87be2e5c7afe7ac2a2 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e5e362eda60d8ebfc4d5734652cc0149e87f5491 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:177ed473a998ad129be7b9858e7cf6ecec36b98ebaa55752d918f2d49b986f40 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3c5b36cc35e75fb4d9251f2f63df992ee0765da --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba501f5f1cc29e876d2011d4c7f160d367ae3bd186fe81232a7e780f20a7b498 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e37f4bdd459f4a892b4ee428c306ccc0a539550b --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:930599a57a222e8627503774f0e8c7ada97682b76e369b26da427af771d2acce +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ffa28fe3c6cf090c90a4d9077b6ca3d3fc3aee8b --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38654836157437520d0d95017b361206d8ef582c466bd61a1d10140bfd1627e5 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ca65cd76eaea5fed39e11674622b5a9ee472e1e --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24f984336df3cc8b323473725153b5973cc7b0c7f4d730352856ee60a73d1803 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d81a9202dff2081bfcf4597672feb37787fda5d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84749b2fb6e5b1c4decd94f97a625edb9190b93b6996b8c033b52f371c133cf0 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..afe4d5bdaf61bcaf4b56073f73cd591838ff36e7 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5f34f688afc394317b76d4233de03432f3815a6d050a7e5c7305ef1b75108fd +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5dd92eac3935c1da860596c64b740442c70b31c0 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c6645694ebb75631821a31595a7d8b4b7bdf527dafc6f0942c77009fc7e8184 +size 199058914 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c132b32aca8afc77c720d21e58e13843d782db2d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f17af07f6aa94c7f7179193b708952665f8fa8756db8215971a0cefd6d90a6ed +size 199058914 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36cb4be3cb884878d67fcfe8ec45bbd9e4bb74b8 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b52b08353abd81666df391605d6b284a0008455f61122437c144084ca504e24e +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..965fdb1c8a64b876f9066f366a5b849081934746 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9063afcf8191efdffc754dd8c559051aaf443a6d00eef6a58f9fad24a7db6fd8 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea8d47c3b3e0943aa1a18cca4ed4ca9ab9d8055b --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e38bfa487334f2a62bf5b6be23869e2bab5ac17396eba3ad7720ce32bf95a396 +size 199058711 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d8b2914d5dca3be00f42f8c29f55bfdeec3dee8 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d164ca93e1a0bc61403eeb4ce7b123b94672474ff4dda5fbf7727fc5721e8ad +size 199058711 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..893fbad95f67c2dd2819396186d05753571f8855 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15878c6f500f5b352de0655cf58801471670efdb569606da24bc001f63ad8a64 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61ed229eb19737711aab45a6966411386e8cffe8 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:299e2db917783b313fa531b95b4b415a8c3504119d40f62e708815284538fff0 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1392a821eb393e667842dbb11d72dd15345b80dc --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef244f83dee3bd9618804cc28136d985478b096eb4ec80de847777f7357e812b +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4ccc20b906651e1be6dc874cf4763486344b8c8 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0abf6be52cc5d860e93c6d8ea1cd088913e6fcf526c25681205bd968529650f +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3cb02c2edc8e31132134ac68aa012f30f661cb2d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb92e73e629e0e1e2f1e19ddc62448818cd1ac5f01f5247062f15a090e67af8d +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..226be68d65a044f60d363ed84237918a9a0e76ee --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2f92f3141b96abb4d2c65162aa26f58a415f505500f2ba361dbd1f42927df97 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f7797d318231c4339df4147e5b8f85128bc29db --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc7ee7adbbc54ccf0b6dc8337016e85ab816bfcf1208e9cff8514b361d8a020d +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5bb2421345063b13667fe002960afb66b8b4ef13 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a7d2406980a5244c4feb6b7642658bf3d80ea6a09c8b5a0481170dc99ed41c9 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7b64a6f61e397e94bad2e350cca7b66ab0b7a93 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad1185748eacac5a5405c1e7f317e33c38c494b2f4abdf3296006b996f9cf581 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ed73d793065a69df32fd2157e602caa3683aa87 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18df6eeebbe0e7474b74dee7f985b6cd078b875f7b05ae5199b560e5d4ff3df3 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f07b571e5b886f98938c5568395507afcfa7581 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8da44416a5cc3efaafd95c2bd27d6e7d4fe8fbf46357b772b42c739de749f735 +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..090db72b943636344c54fcb2752e499bbb4f4c20 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92a7a90e9a6299260ec9b14e6719228446cd56de25ecad9563b5fd524e2be676 +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59dfb37da9f9b6bbdb3127317bf057e4063aac70 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a4c56e19a468e755f505479bee2eaafc018b4913d5593c01a0640da6ffd833b +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e1062447500150514a8afb0ad37880bb6736323 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97903b327b203d99f9563bc5fa1dfa5edd7568eafa1b44400e17f7c85b06fcea +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e161346e9f59532f04bdd7c067bfcdfb7896e30 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5af6b864c6a27f5444882e612f3f75919068f40dd44434eaad3c4e042c2aff0 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4990665ce1e31a08a52c1ac82c5541d3ca4b389e --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da23fb1a89cc7443f60424a270d09e812a6fc0b353be833cb01c3f68067a1ea2 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b173bab167e785f74df00a7bb81a45df2d0d02f --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8420ce4c1262e8c5d01c5b4baf46b1969da9a20bd8a7fc18374f23734871d01 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d045f22d29dc3b54fe3a9eb665c3270fe79d714 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:933929c110fb50a34aed835b261ad1d9be67f8b93b08db7e26e04f8d7d1ade3c +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a57508ff1b966aca341ba7921c2a8ef66ff1ba4 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91623631cba27a4a5248fa2035b627ba1e137a7cfc3eddbb2d1d0a9ccfe787da +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..043f60e117aa7f3e2590f6f0aba7dffffba81090 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9b6c70d93b9308063b60a25e4ba2f19b523b8acbc8b29a7171dc9f2bc3b7b2a +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70241cec73b7c85195b01938231a4e57bbc9f872 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:384a8d94a8970bb18a8e0336360b6f2ab5bd069051d7034584161227f7080893 +size 199058647 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4060ec424a39fe1d5bf73a2845ed86ce7c425294 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92207302cfdfa2494dc5469d3577cf390e1574f5a39a165d145367b5fe741935 +size 199058647 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a98df0700c7beeb4e89dcad79bcb7e0c97aa5c33 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86752d659da8792bb3ad20ac14c63a7e1ffdf5e92e4407107d65d72745e99fc2 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5b586c120e5e49af29ffd75fc4838f3ef930b57 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:075bb85844cab367ceaf67d7f4eff0f4263c6cbac9c50f0387515431f29cac22 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..90e42052c045a2f3f507bf44d680032388ca3e70 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:976d5f1dbb28f8c8597d6185115f620023c1f9f543af4f0005ee58cead7702a4 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..34205e705408f4d45f9dca9cc9b58dc106d426e5 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bff4604d568a6656093574ec2c2316d28bd0a2646cbccf5c108cf075a3bd8141 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..51ff9d5c22fb60ec73432c60dcc694a88db13e11 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd79b9c2ea462daa4c27958b7f6d6b70fc7a817c8dc929dccf3bb6d350aa0756 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a34f1d894d9235e438d3459ac2e8735907d8969 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6108b33d6ef819b1839ee743c0a0435fdd1e391806cce6b23036e4087241f522 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f05c2643e45dbbd378cf49d97ee39fb451a6fbda --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1e3c5b42a6f034fc2594bfa98667a4823ec5a287372a212e967acd17629082e +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83b7892d207a59498f9f0fdba5748631ae8e3e28 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8612e492b394a081be2fe9910d2f65ae0e0f700360f1138d2f9f0d86f1b99aaf +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5766a39bd65962717857448b7a8a0d32c3da2955 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f94128e7ad1bcc94f4d5bc2693fa97a6858b6a2ba78cb9dc0715874a6d21ef0a +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a08a8ece5241a3e2e20765b5405646e8cc21243d --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:143e53a0a7d3e423f21391d22475dbd5285ad74b7c11bbc78761bc93cca44aac +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c13f68c240babc33b226d3cf601d617683ae7f7 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed7bb1e5bfb3e77a95c0879e009933e8a0668018fa241644f17522f119c61651 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98dd4cdcbd84cebffd578b21891bce0e7de22d88 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab138aa86458d5a29521ac34387333487178df5a97e394c4f9ec4e554db832b0 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3151f9f6e575c901969bcd4fa18b66a54578ff6a --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b65bb2be130dab077d0a7e9c4f883853ce9c03e76e937816adf0e610e728edf +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30b046b96f69998e8f51302645a80fdcb401c0e0 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f563c52e2b50b6c22094aecca9ea892eef5cd34d058f2a4ea25de546599bf1d5 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ffb3b636184adbea336b156ba4889fd578c1381 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c3087969ba481abb62795a9c2f1b2dd544fff919f443b9e2c51b7b84cba0208 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37489609ddd5fd8d95025d076636e3f181e4d6f1 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:115b706d79e9753c64c45bfb69a9f439fb0ec26abab4b35db714bd7d7bdb4a50 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..413b9087b0c794b2f5b6d7c8198ad9743c6bcda0 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b06802edd7a5bcb8ae1b144e6b4ae95e3e87d6d1cd8ae11b7a05eef16de0869 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05c810d3d973e143d05f11d83242264661541246 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1958374e7714138bec837d2809102a303f4709bc36554f5019ad926e6178b28 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f06db703dd751274ff2adf4e1f4c35dbb0a3d955 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f69f48a23d9ad9a21325448c95f13f7f0f12f933184f2be2d32e0efb795a898e +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1a6d66ddad7453b995616f4566e9d5b93cdfbf8 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c51b4d6d7b0fbd609c3f7ddb523bfe11912d105ff1605a67ecd145aa0c0dcd1 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..522526cddea1c2f5938a94f92f49550457f18d0e --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6b3ce6620469c461fc7376fbf700eb7fbb68e0a1c76ccebed05289632162f87 +size 199058647 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e76cb73659615eb579b961002e74fc91e109ef3 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0922d6505468f5356ab9625b140366fe96e04c20a3ae8e506d38c545ea6e5c1 +size 199058647 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f781ea09f5afd2b0ba7989c95dbd27cfe6c25d0c --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed7c23db69284e0c5a25ed461a25cddac2195021fd16b793480e7dc85753263f +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4adeeb241f459ad44c195e01c707bfa5f71c5d14 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bf293afc954fe7f738ec2a8fab2ee0414502b35ad5c2bd340abb8a0ddb4f2b2 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c0408cf9dcd8926f35388310cc2d3661b152068 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76225a870a1b16ada7a6109c8ad5a56f02788aea1935bc64aceb3b28342b4916 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..098e1615a26c5daf44cd87ace19b98c074ce8005 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d4a692d491b80fa469cd41c267c2b771088ba6a8a5cde3b2c88ec8e1ba9b583 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..340bbe51d902959f5c45a0668f2aa3648abc0a84 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87ec8be99c4811b91d0c14b5aa94d0cd6e0aba7c9779e5f70dbc3857fbb5726d +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9336c09ee11df11fbaf1d0e34e796960d3b87eb --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d21b651d338972ed2031c6e9120d7f11612023c86f28649cc50c48ad34bf2bf7 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00a1c5b4a3fdf7cf04317da79677a15268ed930c --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:086e27f6b1c2e1bee97eba812a01a58f2b558712da3f39aea38e6fc3f7eabd2a +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5fef25d536ae32195c13e9d842b82dda77a75562 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1768a1371a043639963c0fa64b349c843fd233c19b0f334090f4ddadef17972 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9564cf09c5d76a9cf8e3a8dcd9f2f0956d5c370f --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c526de924ba1c3d03522fddb33ffeca0972e6a98738c64b06a594c4584b6661 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6bf2dfb3f20aaa516c06de925111fcd1944a7d8 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ddaaac54b356cd3d970e76c3c9bed1d5fd0f462af65e4c2f26f9cb764b4ede4 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab960bb7e23592b5768bbe2a3eca14086fd2cd5a --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53aaef7cb5f0c2d86d4922866dbc199abdeb585be264119c5c60da00dade0a81 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af78f72042ad7cc0c4a6cf4b01498ab8debfc6a1 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c16579dff0424aa8c27796ff23c2f0e444ea93af5c588de16efa63c71cffd002 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc56b0faa559bdf204a87ac2e48b82cd400bce59 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9e7a9e4a2d3c7ece5ea2733c294ab5299d69faa720af4ea19760d1c1e16be09 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c2d10bb7a5c8120bdde880a27b1f80406283b6b --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bbd632b9137080dcf584b8d3bc7083677ef42b4099205f37db53a0f37c824d3 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..74ecf8a5d59039f34d0e350a1b8881e4b9b36695 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdd28d9de3dff20db0e28442b573bf344550c00b7223f8479382b663bfb7788b +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d03eb1fa539225e79001de39b4f938e20968333c --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9081ec1b79048f40f07553ae576c9e28ea23d7cd5831a905aef1374a9f3319 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f329b5a43935c188912d7b6b34660168fd6974cf --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f2394ba42f4eceaef73e16fefdc4844e9c5f6520faddfb9e109fd89c8ff2e5e +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..929fec8029c26bc79c090e5836485d7bf2940bc6 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd9608bf08b23b20ff63ecff34cb6a16baa6918ea216c7319a857ea371033de7 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..775f6c48de9c5b610f52d3e7e7a850c8d29187fb --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:073cf979f791356d36de93c4844d27633e2cea97edd54ea69ec4a62d69852683 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5816f4bb7c9e0808a0016d2defa55840a6f683ef --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0565d8d56c8792d197accead75785b173c88de298719d5ef7f4899bc55018ddd +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52b07bfd62f2eb2f02ceeae7dd16cbc74d049946 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11134c44762b3027006a44ba5850b6397f3b10c9499e24c831b29b1b59f62132 +size 199058647 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2939bdb898448d6d9cd86995cc5f5386c9a6b007 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72e21d2f9910eb04a53bc72a255373527c7b25b063ae5cf5bf43271f238428a4 +size 199058647 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce3200c5d65b0589b23c1ed7012619a8bf51cf63 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e21abd76e43e82fbf68c64bdb2c87aada03696e58e48b7dd58a8b8f7ff897b9 +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f69eb70530a90361f8d7e1b35e91eea0bee9e30 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:225c728b955fa13f1d7615715e2970c1dd548985c6ab4ab77afea74ea84873b3 +size 199058850 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..699362e81eba3941a4948af9028daa49e623bc91 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca49187defdbb528ef060c151eed45d75ff6754b77ad2580b113b2f94afb64a3 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5cfb91964796a46a17f6fc27ec82a07498a4117 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9afe843114bc76a6b0d03a850e1cd20dd74568e7a13e04257da3b5bd99f9358 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..66c25a08e583953672ba40c8c7e1d82113935926 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a12d4f46d3b207a567fb90bce8ac0aa9d9a20c678a651eadbdc09e5038ad9c5 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a18238435df542641d873e78ec40a389f4c4524 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bafba188098e0f1c03513e389d431c9704e0f775a3c9ff9fa8bdc0352bba6b56 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3ca269e09bf6543d01145b07ad3c450e6274200 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d594bd65a9b68198b6d8ff2bb25034a39ed987f2918115005ea26e64bcdbf1c +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2af4e902799f258c2447453bfab0366759d19c9 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b38187b0775d6339961417977720c8444b2f399b7e44cec272d03981aab84081 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6db4bdc8260cee0ab0a52dd1c1bab067ea3bd888 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de9d698723a3b5db711d6a94b3be4d4c0d1b888a9996b63fbb91aa823f3d45dd +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..867d017069f318f5e277f20c8bf185a264840473 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7544e06a558de6a91c11a1e21a1db8fb64d37636e858647be781c4b6f147b463 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8dd4b551bc555b96c2dd771271552e54c4a816a0 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c09b639e3c231ef402bb07ca5128afe9c6c25346b435741b9a3fda3dd0c51044 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37f7bf1cefb6a8224e12ce8d9c293935fb624fa7 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8c1fa488d380270a36760fdf3d523297cce3b1f396c2a0ca1039d76e48b5bbf +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c4d8c422d26ecb07657edc5b4ce1367d4d86a1e --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3cefaa4b03737f171dac1ba45aa30310c2c1a5283741ff6fdebfd0a3861fe99 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6d34f3225b097eb30c4be0b6525e00de66afe34 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e198e0dcfb8a3727965b51829334aae1612339a9568a1a205fc39e7519758dc9 +size 199058722 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3105499e8350b5a42357554c53601d73458cfdd2 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:632c604c0604860e80fc4fdc8de0f88736fb4be76551dc3c1698eabc6af35a42 +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a99cf12a69cf608c602e82d971e7740c2288e010 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ced91fea88b89ce68be76fa96309c8370cac4dfffdc4feaa4181f18a8c7dcad +size 199058786 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bcfd74dd59177e062512743e199afa69e5237ac7 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d6bc34fb1841e1144a8cc4e1ce05d8b0e0dd9444a499fef7a5c0167492f4eb1 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2fd48dc0def3264be0af10a38e314552cbbf3ed --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99649dba5feb3e59a371311a80b82d0014b0a85d6e3aae2e390b443e7334b1bf +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d699a98129727bdf34333500b78a9f59c4b0800 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da5e86a5d2f691103456a11ba58d9f048a97a6c64b624acae66f187f15962a38 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4cffaa00ea1c431346d462613b97976fd526ece --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0588df9ee9d8cb53d2f77147e471f75b8e6fa71a4a73368ccceaac086e035c54 +size 199058658 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f448297cd91fad382c3102ca6c599a389eb7b732 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9460ebcc19e8e2b59d469cf357cf60e996b5098ff766dd7458ded4121c5742d4 +size 199058839 diff --git a/perplexity50/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/perplexity50/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..743835e47697bc30eea2e492cc6852e0ba8a6617 --- /dev/null +++ b/perplexity50/global_step80108/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db9317e779fa830ec8f3a64afb66ac4d2b3a0fe159529ad3494165b44397467b +size 199058839 diff --git a/perplexity50/global_step80108/layer_01-model_00-model_states.pt b/perplexity50/global_step80108/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..faef1617b067a477f92d05beafbd0c4c5de2644c --- /dev/null +++ b/perplexity50/global_step80108/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:634fbee0c49461f081d12c1cd569bb2cdd3c4d76c7848733a4af41b5c3f29734 +size 167511299 diff --git a/perplexity50/global_step80108/layer_01-model_01-model_states.pt b/perplexity50/global_step80108/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ee949fbe96e15caab7a4cb02c56aca4ca4f4c34 --- /dev/null +++ b/perplexity50/global_step80108/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca08152fdf6c03123f94b86dbeb4d4ac14ca5d92bd94e817c7d8c4bb5f7901bc +size 167511299 diff --git a/perplexity50/global_step80108/layer_03-model_00-model_states.pt b/perplexity50/global_step80108/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1eff0ad53b24b97111a581d3bb06843db194d2a0 --- /dev/null +++ b/perplexity50/global_step80108/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38e9cabd57c0de5b08a9d50e09871ca988495439650e15534b2c80b0dae289d9 +size 113308931 diff --git a/perplexity50/global_step80108/layer_03-model_01-model_states.pt b/perplexity50/global_step80108/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1657ca6ccab1668332f213f67540aacb1c7268dd --- /dev/null +++ b/perplexity50/global_step80108/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b45693cc709abda59c7540d8f39cc11678be408b07c5c524a687b8005f920b0 +size 113308931 diff --git a/perplexity50/global_step80108/layer_04-model_00-model_states.pt b/perplexity50/global_step80108/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98bc08d382e26957f16e5f5f75f6ec75ec0633c5 --- /dev/null +++ b/perplexity50/global_step80108/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a07da2c6cb757ea3be86c88ac1a1b1bc4f0be85e60b4c3317e96507f4bda52fe +size 113308931 diff --git a/perplexity50/global_step80108/layer_04-model_01-model_states.pt b/perplexity50/global_step80108/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2e57c6bcd2594f4d75dc2589ad5cead5ee5b91f --- /dev/null +++ b/perplexity50/global_step80108/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a928e0df40777a668a72818b42d61d036a22acbaa41d84f8dd2afbb639b9f02 +size 113308931 diff --git a/perplexity50/global_step80108/layer_05-model_00-model_states.pt b/perplexity50/global_step80108/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3fa3c05ce1b07a39d59dd110a5c601d81919e258 --- /dev/null +++ b/perplexity50/global_step80108/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c78c617096701be50ea8cee8b2b1935b27e8543c3c72692359aa10694d0e1354 +size 113308931 diff --git a/perplexity50/global_step80108/layer_05-model_01-model_states.pt b/perplexity50/global_step80108/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e82e66a93da1f0df2673dc35f5f5adfd4e5993c6 --- /dev/null +++ b/perplexity50/global_step80108/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ba98e86e60173eb56eb7813c1a11104c998b67fcadc04b4e9babee26c7dbf39 +size 113308931 diff --git a/perplexity50/global_step80108/layer_06-model_00-model_states.pt b/perplexity50/global_step80108/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..78674f041e06962244d2b2dd800bc64feb1e4cbc --- /dev/null +++ b/perplexity50/global_step80108/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:925152fe163504c16dde1892a4328b400f35e3596198ce9f41c2b0271c1e85ea +size 113308931 diff --git a/perplexity50/global_step80108/layer_06-model_01-model_states.pt b/perplexity50/global_step80108/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..da83e900ef4863eb2637b15884062c435cd93fd8 --- /dev/null +++ b/perplexity50/global_step80108/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c60b7f81dfb1189b25cd81b9dfed3785c238c675acbe3d163ed62a4781d38bba +size 113308931 diff --git a/perplexity50/global_step80108/layer_07-model_00-model_states.pt b/perplexity50/global_step80108/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5ebe5dee83fb3e83dedf1751a5bc2f966b14a3d --- /dev/null +++ b/perplexity50/global_step80108/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c04b59551226d2b87e918dec7c104221ceb18f745ab4b7a2069153ff2875e158 +size 113308931 diff --git a/perplexity50/global_step80108/layer_07-model_01-model_states.pt b/perplexity50/global_step80108/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31f1d9cfcc3163fe7950aa58e0adc8f35d62f913 --- /dev/null +++ b/perplexity50/global_step80108/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a6b7c41c424cb9106e459f014f93170b030948135afa6397b427c9946348231 +size 113308931 diff --git a/perplexity50/global_step80108/layer_08-model_00-model_states.pt b/perplexity50/global_step80108/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f43c7b18230ee004484eb07bfdd55b163927409e --- /dev/null +++ b/perplexity50/global_step80108/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f07529e4943d7fc97b26d2e1d773640200e048bc199acc4846bc54acc019ddd +size 113308931 diff --git a/perplexity50/global_step80108/layer_08-model_01-model_states.pt b/perplexity50/global_step80108/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a00f52a34d3bca531025da7d710cef46b0247612 --- /dev/null +++ b/perplexity50/global_step80108/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d236896b8235505069c613b539000f2dc0649830baa705032c4534bf08caaa2 +size 113308931 diff --git a/perplexity50/global_step80108/layer_09-model_00-model_states.pt b/perplexity50/global_step80108/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d68267c85cf2d4b0d61f75ceea137aac8efdfc0f --- /dev/null +++ b/perplexity50/global_step80108/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c097050ff4a68d82e3cfb3ec064c0dad7f7c65f7e65c3e70f4cb992e2319bd1b +size 113308931 diff --git a/perplexity50/global_step80108/layer_09-model_01-model_states.pt b/perplexity50/global_step80108/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a890e627e04d555d8ae4304d5b27abdf57ef29b --- /dev/null +++ b/perplexity50/global_step80108/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b9f911d68ed5b9f76131be2db4b66251a5be7feed19a64496b10228561a5190 +size 113308931 diff --git a/perplexity50/global_step80108/layer_10-model_00-model_states.pt b/perplexity50/global_step80108/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9841e2ec8248f0752b7dcb01d870e2e4d2502cc --- /dev/null +++ b/perplexity50/global_step80108/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:568396f32f09a1370dafd0f820f36c68e330cc92c42a77dece77b34d6d8a3076 +size 113308931 diff --git a/perplexity50/global_step80108/layer_10-model_01-model_states.pt b/perplexity50/global_step80108/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcfdd6e1178faa54c2738d861be43c58b4b7fbaa --- /dev/null +++ b/perplexity50/global_step80108/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8c7c02417d5c28013cd12367aec709c2846719c7b4d0f28cb5d0c7af7697436 +size 113308931 diff --git a/perplexity50/global_step80108/layer_11-model_00-model_states.pt b/perplexity50/global_step80108/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e4020793114c9eebda3113e7f00d9153630793c --- /dev/null +++ b/perplexity50/global_step80108/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaa0aa1541ed9e864fa357fe55d393b0cdcb3526928a750d6560b5572a7cb4c1 +size 113308931 diff --git a/perplexity50/global_step80108/layer_11-model_01-model_states.pt b/perplexity50/global_step80108/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..60fd1f6acc838ef082f1707d2a1e32a4678b8ab2 --- /dev/null +++ b/perplexity50/global_step80108/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f503fc3b923ab3d2336f4810e60078dd11e52436240b7315373935615689d132 +size 113308931 diff --git a/perplexity50/global_step80108/layer_12-model_00-model_states.pt b/perplexity50/global_step80108/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ecd09b8fadf047a0567b1ffcb4808b99b8544d28 --- /dev/null +++ b/perplexity50/global_step80108/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:350f54f7bfdd0b601cab04a9055e25c2d3d2445ff8ec01d34cfb379dbb85ba96 +size 113308931 diff --git a/perplexity50/global_step80108/layer_12-model_01-model_states.pt b/perplexity50/global_step80108/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a9a3ed3ed0fc917e38931b2d0b19e5b4b7d75d8 --- /dev/null +++ b/perplexity50/global_step80108/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9d915d51db3c93433643ed4583d6be1e4bd0518ca510e45d12da0acd06b811c +size 113308931 diff --git a/perplexity50/global_step80108/layer_13-model_00-model_states.pt b/perplexity50/global_step80108/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62366497c032f5f23f8b1ed5acaa926e87c88055 --- /dev/null +++ b/perplexity50/global_step80108/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8488f98c4ad9eaf957a7b96d4c4861dc3ab85e1f75115d992748cb8604f71f23 +size 113308931 diff --git a/perplexity50/global_step80108/layer_13-model_01-model_states.pt b/perplexity50/global_step80108/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec9917e99317fd264a28f62507d1c1ae738707d3 --- /dev/null +++ b/perplexity50/global_step80108/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b87cb5de7da17e3aaf7b34a9d1b100031304b151f754e5c488ae69e34f824e7 +size 113308931 diff --git a/perplexity50/global_step80108/layer_14-model_00-model_states.pt b/perplexity50/global_step80108/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e122e4f3a0ed255e9e493650683eb245d0a7b9f --- /dev/null +++ b/perplexity50/global_step80108/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b4197c7d3059eb4ad7b059b550b3181a8f82e5e407a70e8462ddb8696f2bc6f +size 113308931 diff --git a/perplexity50/global_step80108/layer_14-model_01-model_states.pt b/perplexity50/global_step80108/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec16945bbe78f7255bb56f39f7785a7487f82d18 --- /dev/null +++ b/perplexity50/global_step80108/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0351837819d7aba04c8549a3a08fefe00ba433a34e073ec6ed0d72e0aab35204 +size 113308931 diff --git a/perplexity50/global_step80108/layer_15-model_00-model_states.pt b/perplexity50/global_step80108/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2143cb9914720cac8fca92d74a3ffd0fdbe5246a --- /dev/null +++ b/perplexity50/global_step80108/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b156470be5eed88f64c9f9f9e926c96f6e47aeba1a5475d0b7d521b35fe32f57 +size 113308931 diff --git a/perplexity50/global_step80108/layer_15-model_01-model_states.pt b/perplexity50/global_step80108/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..691150b9b445b2c9b2442e67c469dd5614f49836 --- /dev/null +++ b/perplexity50/global_step80108/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5724c201dcd23b7548cb285d23079789a0fa3aad1235b3a74468fd96c5a0d7f7 +size 113308931 diff --git a/perplexity50/global_step80108/layer_16-model_00-model_states.pt b/perplexity50/global_step80108/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..feb95cbdb2555aa9977b09dfff475469e771f79c --- /dev/null +++ b/perplexity50/global_step80108/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26caa96eec22d5f7da0ad71a390de01596c538bfc30962db0daafd601eb7773f +size 113308931 diff --git a/perplexity50/global_step80108/layer_16-model_01-model_states.pt b/perplexity50/global_step80108/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..435cbbd4555fd208aecf563021af4cf13aca096d --- /dev/null +++ b/perplexity50/global_step80108/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74e06e7a15760066d5682383895c93d82ba330f9b4d691690d82345a22bdaefd +size 113308931 diff --git a/perplexity50/global_step80108/layer_17-model_00-model_states.pt b/perplexity50/global_step80108/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6458f64b899f9ba004db69044d173e1b8b13e684 --- /dev/null +++ b/perplexity50/global_step80108/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb75472367a88078b4c79c43f92120924c73dfe2c19e61cfc3fd142564dd9458 +size 113308931 diff --git a/perplexity50/global_step80108/layer_17-model_01-model_states.pt b/perplexity50/global_step80108/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e516dedaaaf9fa1131df29aed2ca28c75db0394 --- /dev/null +++ b/perplexity50/global_step80108/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e66aac43c595cf0b28f67a9c8bda8658e7c011d2e2da1df8e225c1be4ccedf2a +size 113308931 diff --git a/perplexity50/global_step80108/layer_18-model_00-model_states.pt b/perplexity50/global_step80108/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7f30aff55c01da610f56a2d5b4797f9ecbe24f1 --- /dev/null +++ b/perplexity50/global_step80108/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:927d4cbd5147f28835a9367df02c7d12061d59681217d83af6c8a7a0d53652f2 +size 113308931 diff --git a/perplexity50/global_step80108/layer_18-model_01-model_states.pt b/perplexity50/global_step80108/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e76b546e7ba52669598c7801d5e3f9e0b0d99793 --- /dev/null +++ b/perplexity50/global_step80108/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a4c9a992f4d2d9adbd792c5fb0b8e9a564ef448e36dabc56b0ef17f3c8d76c8 +size 113308931 diff --git a/perplexity50/global_step80108/layer_19-model_00-model_states.pt b/perplexity50/global_step80108/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7bd2e3c81c95fa61a5cb859dee7bb4d26b61533c --- /dev/null +++ b/perplexity50/global_step80108/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d66c4329855d2455bc205beccf6c615c743c2545e1207d69fe13b2e13a7c888f +size 113308931 diff --git a/perplexity50/global_step80108/layer_19-model_01-model_states.pt b/perplexity50/global_step80108/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fab6020b0e8f790c2a5a6aef6748bc1f128bd6f6 --- /dev/null +++ b/perplexity50/global_step80108/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d9f125b3dde8413d196de716cb2c9e6fd44bfc86772876e075841dc03905564 +size 113308931 diff --git a/perplexity50/global_step80108/layer_20-model_00-model_states.pt b/perplexity50/global_step80108/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f71b9451367112752568b29d0ca50946c3e590f1 --- /dev/null +++ b/perplexity50/global_step80108/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87bd6ec0de92f1597de6630826bbb470f7fc174effe709374d011ed1554001d1 +size 113308931 diff --git a/perplexity50/global_step80108/layer_20-model_01-model_states.pt b/perplexity50/global_step80108/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..efd3e3a54fc4e03e1c065b0ac376d35f1b473c10 --- /dev/null +++ b/perplexity50/global_step80108/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b812b0c6d629a566f7d5d24e5b2c7a2a680f286baf66b9eba9896e9e65fc259 +size 113308931 diff --git a/perplexity50/global_step80108/layer_21-model_00-model_states.pt b/perplexity50/global_step80108/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63beee4c5e4e0427034f869299ed875fec8fbc10 --- /dev/null +++ b/perplexity50/global_step80108/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:010403a197e1775ad42fe25a5b717e7f40fe34609834846cc37f6bfd8e2a958d +size 113308931 diff --git a/perplexity50/global_step80108/layer_21-model_01-model_states.pt b/perplexity50/global_step80108/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..799a5213e46229e605bdce6e7e16a62426a4e844 --- /dev/null +++ b/perplexity50/global_step80108/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2bbb4ee09fa7b697c6cfad42f7614033bb5279ca40fd156949f16504510bde1 +size 113308931 diff --git a/perplexity50/global_step80108/layer_22-model_00-model_states.pt b/perplexity50/global_step80108/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29e8ce11da840f4bf9240fa3d7c0c87ccb3f1ada --- /dev/null +++ b/perplexity50/global_step80108/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58f887db1d54044e2510e0d3fd89390c42941a8c71aeaab11dc4010e336be133 +size 113308931 diff --git a/perplexity50/global_step80108/layer_22-model_01-model_states.pt b/perplexity50/global_step80108/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ff7b7f15039c01ca005cebad6bbc85a001d6e0b --- /dev/null +++ b/perplexity50/global_step80108/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a6031437e57122a8feea1d9a6c01a4a87be7583d1315e238ec57fec06acec83 +size 113308931 diff --git a/perplexity50/global_step80108/layer_23-model_00-model_states.pt b/perplexity50/global_step80108/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6d2a2a3b73d4752d1631dcd069eaa007183adce --- /dev/null +++ b/perplexity50/global_step80108/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0b037fdd107e241641a9a6a755a26adec3df19bb2abf7a5c714a0f3c57309cf +size 113308931 diff --git a/perplexity50/global_step80108/layer_23-model_01-model_states.pt b/perplexity50/global_step80108/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..68ea9bb66c2d4a436fc8e2c6bac88dc1843e64fe --- /dev/null +++ b/perplexity50/global_step80108/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3629880808d09b0d3db74b619158bf3046137513dee58e534d7174a57a0774d6 +size 113308931 diff --git a/perplexity50/global_step80108/layer_24-model_00-model_states.pt b/perplexity50/global_step80108/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..79b131e4a592cb3a46533d926f5138b196c43274 --- /dev/null +++ b/perplexity50/global_step80108/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e1fbb16ae2107767247da064e5c71bd787d797b2b9d8e94d50b23f9f877933a +size 113308931 diff --git a/perplexity50/global_step80108/layer_24-model_01-model_states.pt b/perplexity50/global_step80108/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6a9198c38be85a862e1dca3d90a591e20963482 --- /dev/null +++ b/perplexity50/global_step80108/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8321d74512e1348a6450fe764f841c30d515818b1c3a64164340a63729840d3c +size 113308931 diff --git a/perplexity50/global_step80108/layer_25-model_00-model_states.pt b/perplexity50/global_step80108/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c039bf995a64fa688eb9115366985be21ec5981 --- /dev/null +++ b/perplexity50/global_step80108/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c6b77d71b3673c3680ec7ba04f92b2cdb4effabc9ec46d4b1efb6d23b072e29 +size 113308931 diff --git a/perplexity50/global_step80108/layer_25-model_01-model_states.pt b/perplexity50/global_step80108/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1346eb0e142969cd01d1a0dcfc884a19d9fb844 --- /dev/null +++ b/perplexity50/global_step80108/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd0d352baa2c977743a115d71544504c27341d2d14c80284e65bd1fd15cdde7 +size 113308931 diff --git a/perplexity50/global_step80108/layer_26-model_00-model_states.pt b/perplexity50/global_step80108/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b2ae16422efeb787b6735bb843aa4f74f76aa84c --- /dev/null +++ b/perplexity50/global_step80108/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcb6a98adca5d62a330657559bbd9c85d756445ba1db5b79fb560427575d3565 +size 113308931 diff --git a/perplexity50/global_step80108/layer_26-model_01-model_states.pt b/perplexity50/global_step80108/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e84e5d8572cb84f0de9aeb8bf91dcaae6a09aeb9 --- /dev/null +++ b/perplexity50/global_step80108/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adef9a05d37586ab2055aa41c9ff8ca57f0692979027525159f9b6c0af1638a2 +size 113308931 diff --git a/perplexity50/global_step80108/layer_27-model_00-model_states.pt b/perplexity50/global_step80108/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f7cffc44f9182f961dce0b9b930a46da078e4a4 --- /dev/null +++ b/perplexity50/global_step80108/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:146350b10d4784e6b06ff0ce274bddbc627ce8752877c005a5954e92d22d1911 +size 113308931 diff --git a/perplexity50/global_step80108/layer_27-model_01-model_states.pt b/perplexity50/global_step80108/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe6b89d24a43169597ffb2505ab9e06fedb87ffd --- /dev/null +++ b/perplexity50/global_step80108/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:772e58f6727c3b7b2fd1e8ac0d1bde565f6e4f60c246b1ad80692ac11f0221e7 +size 113308931 diff --git a/perplexity50/global_step80108/layer_28-model_00-model_states.pt b/perplexity50/global_step80108/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f8f4b4e9711f198ded56c0f03f8953c4d402e52 --- /dev/null +++ b/perplexity50/global_step80108/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:012162e29ae768fb96741b2e5916045470ebf634809ed4caad3c7d7f7ad4fb06 +size 113308931 diff --git a/perplexity50/global_step80108/layer_28-model_01-model_states.pt b/perplexity50/global_step80108/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0fcd559251f0e67ecdb48f446328461fa38267db --- /dev/null +++ b/perplexity50/global_step80108/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25f9b78979da25d06f9fe4b450b41098a01edad55ce47269e383db8b4202c6b6 +size 113308931 diff --git a/perplexity50/global_step80108/layer_29-model_00-model_states.pt b/perplexity50/global_step80108/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b0b903ec6f68ce647e75513b1cd480cd62f5466 --- /dev/null +++ b/perplexity50/global_step80108/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98c3738530dc281f5c6a536f137c31c3795eb35a5419daaf4c6f909d168bbe1c +size 113308931 diff --git a/perplexity50/global_step80108/layer_29-model_01-model_states.pt b/perplexity50/global_step80108/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9535b8267ecb928f02ced3b1e3338082cfba6308 --- /dev/null +++ b/perplexity50/global_step80108/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:280b7f074b976eb187201eb66a866a8eeb8282108aa7dc09571a1d820a4e9a79 +size 113308931 diff --git a/perplexity50/global_step80108/layer_30-model_00-model_states.pt b/perplexity50/global_step80108/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a564b22af9eacaf408dcbcdd8c04df7ba88db0c6 --- /dev/null +++ b/perplexity50/global_step80108/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cd217ed647b11f010c70d433e14103277971dcf387082e10101e24c6a3a6be7 +size 113308931 diff --git a/perplexity50/global_step80108/layer_30-model_01-model_states.pt b/perplexity50/global_step80108/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fbbae62243536c7a839b64716a2e14b9ae8134e --- /dev/null +++ b/perplexity50/global_step80108/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05856019cf84bada80c1d2d7ef98607d932cd85986510ae225c3a99a5a93616b +size 113308931 diff --git a/perplexity50/global_step80108/layer_31-model_00-model_states.pt b/perplexity50/global_step80108/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..72902e3423063b3196a162b383a94cf297158944 --- /dev/null +++ b/perplexity50/global_step80108/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9ec8654c605b4d909fb1c96d1f9ab3bd28478b296ec8bc3cbeb745ab5132c56 +size 113308931 diff --git a/perplexity50/global_step80108/layer_31-model_01-model_states.pt b/perplexity50/global_step80108/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa508a33008f28e6a7d81ca2e5ccda2e684d8f40 --- /dev/null +++ b/perplexity50/global_step80108/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b6eac877b81441906125d29cec2c5e58fa2c55b5b6fe1234ebcd255aff52109 +size 113308931 diff --git a/perplexity50/global_step80108/layer_32-model_00-model_states.pt b/perplexity50/global_step80108/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cafb7077f6d969706137dd4f32c97b3140a8f033 --- /dev/null +++ b/perplexity50/global_step80108/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ace2c642ecf0b8809916256dbf2f51c23529e1f168785aa9071d7fc4cba9573 +size 113308931 diff --git a/perplexity50/global_step80108/layer_32-model_01-model_states.pt b/perplexity50/global_step80108/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbd23abf32808111cdeb32a5592526f9423e89f6 --- /dev/null +++ b/perplexity50/global_step80108/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fecb893ae0889ac6363140bf925f6502d240959be4eced29178c4051967b7b5 +size 113308931 diff --git a/perplexity50/global_step80108/layer_33-model_00-model_states.pt b/perplexity50/global_step80108/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05fdbe4ac30a43c77b93e54db9e52da3024dc398 --- /dev/null +++ b/perplexity50/global_step80108/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65ce923e18c6daa2e833964f7e402f251e3fb623b9c9ed5e75e7eb868091090e +size 113308931 diff --git a/perplexity50/global_step80108/layer_33-model_01-model_states.pt b/perplexity50/global_step80108/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a23ea44ceb4a87ff2027542ed0ad2ff4bf91bd6f --- /dev/null +++ b/perplexity50/global_step80108/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:364327ea798c55d49bbe236d36f465a74ffab88373d6956f09299e5e2808d560 +size 113308931 diff --git a/perplexity50/global_step80108/layer_34-model_00-model_states.pt b/perplexity50/global_step80108/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..15dc6f5da2e6528942f8cb55a5aed1ec60dac415 --- /dev/null +++ b/perplexity50/global_step80108/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b52d166bd8e6de3118f8233362c78fd15dc0f05bebde685d161c907687acf5f +size 113308931 diff --git a/perplexity50/global_step80108/layer_34-model_01-model_states.pt b/perplexity50/global_step80108/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..640070a95936b43f0c165556f51d676085a65949 --- /dev/null +++ b/perplexity50/global_step80108/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aca8522334d2856650447fe217a6b8182d78659625c1182ab1d4217591cbf855 +size 113308931 diff --git a/perplexity50/global_step80108/layer_35-model_00-model_states.pt b/perplexity50/global_step80108/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd303797548a0dc39d1849668b1668f808c94f25 --- /dev/null +++ b/perplexity50/global_step80108/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dec430b4b2c49b1c4a7d301e1ee632c2d611d267672a12be163f7bb4a732f443 +size 113308931 diff --git a/perplexity50/global_step80108/layer_35-model_01-model_states.pt b/perplexity50/global_step80108/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a699b91244a1017fca5dc124ebd681324cb940af --- /dev/null +++ b/perplexity50/global_step80108/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b0b5eee61a4a81a953f040c3b2ec8b59371ac21bcdac399e8441c0ead75f486 +size 113308931 diff --git a/perplexity50/global_step80108/layer_36-model_00-model_states.pt b/perplexity50/global_step80108/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a8d9bf3d159756513b538592a7f53ecf6d28387 --- /dev/null +++ b/perplexity50/global_step80108/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13b12d747220d096fce869d6c292ce6e937119e02c75ab113b0651b4feb665bf +size 113308931 diff --git a/perplexity50/global_step80108/layer_36-model_01-model_states.pt b/perplexity50/global_step80108/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aab9f7dfb10429f318509ab345c99a77b29be459 --- /dev/null +++ b/perplexity50/global_step80108/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7de8d469d4f30245fa2a2fd8d4ff4198b2bfebf0e0edd9e6a3bc751ec8df390 +size 113308931 diff --git a/perplexity50/global_step80108/layer_37-model_00-model_states.pt b/perplexity50/global_step80108/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6905029a8c1df71f5da6c7c9120b5f41b187ab5d --- /dev/null +++ b/perplexity50/global_step80108/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e1d0068b02c7d3e56d5624cc4c32a4fdd48fa0a68e915afa0cd516729be8266 +size 113308931 diff --git a/perplexity50/global_step80108/layer_37-model_01-model_states.pt b/perplexity50/global_step80108/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc9458030ca67d0b49611ca191e883f44c159494 --- /dev/null +++ b/perplexity50/global_step80108/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abcca4673b9666bfee1ce10a4223732b35cd8bfdd17964562d956f61760c0076 +size 113308931 diff --git a/perplexity50/global_step80108/layer_38-model_00-model_states.pt b/perplexity50/global_step80108/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94478d7087a074a47336311011e4f1deca30f5b4 --- /dev/null +++ b/perplexity50/global_step80108/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bbfa5d4b5cbeb6fd05cee327a924cda532b888a1e0b868133ae1950d57f9fa2 +size 113308931 diff --git a/perplexity50/global_step80108/layer_38-model_01-model_states.pt b/perplexity50/global_step80108/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4a0f1c6926eba7d73ef61ea84da1db6e678647c --- /dev/null +++ b/perplexity50/global_step80108/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b6d0022baae59c401e514d33d37de086adc8c0c29fc9b6862349f56f040b674 +size 113308931 diff --git a/perplexity50/global_step80108/layer_40-model_00-model_states.pt b/perplexity50/global_step80108/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d66a0fb979bd09611272c8ec1d48ac74d93e437 --- /dev/null +++ b/perplexity50/global_step80108/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c257b29ac35ed25a1828fde33b53acb56ece1005cfdc0bb708504c628b541185 +size 13507 diff --git a/perplexity50/global_step80108/layer_40-model_01-model_states.pt b/perplexity50/global_step80108/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dc11b336e2945c720021847de57f9a09dec6efa --- /dev/null +++ b/perplexity50/global_step80108/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03d30e451323bead48d69b603ac8390599c7618f54bfb898bffff1192bc95bd1 +size 13507 diff --git a/perplexity50/global_step80108/mp_rank_00_model_states.pt b/perplexity50/global_step80108/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc76aed657550dde856187f6a5a7f895cb505827 --- /dev/null +++ b/perplexity50/global_step80108/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53a3c7a086d5131aa48dd77940696585423c9034efe841437f20fc89a156fbc0 +size 51507 diff --git a/perplexity50/global_step80108/mp_rank_01_model_states.pt b/perplexity50/global_step80108/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8eb85dfa06bf5b681b04dc28d4d4cae2198e05f --- /dev/null +++ b/perplexity50/global_step80108/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6beb83d731c53b1ff19c43e692b076716e6d1f7fa81cb46a0b12f069222b55ec +size 51507 diff --git a/perplexity50/sbatch_4b284bc4perplexity50.sh b/perplexity50/sbatch_4b284bc4perplexity50.sh new file mode 100644 index 0000000000000000000000000000000000000000..2afaf382cdc069b91a2193d9711314a905377212 --- /dev/null +++ b/perplexity50/sbatch_4b284bc4perplexity50.sh @@ -0,0 +1,163 @@ +#!/bin/bash +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=40 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + + +VARIANT=4b284bc4perplexity50 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +DATA_PATH="/scratch/project_462000119/data/c4perplexity/gpt2tok_perplexity_text_document" +TRAIN_DATA_PATH=trainperplexity50.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4perplexity/gpt2tok_c4_en_pplx_50_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=2 + +MICRO_BATCH_SIZE=2 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_4516M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=10000 + +# Tokens: 84_000_000_000 +# -> Samples: 41_015_625.0 +TRAIN_SAMPLES=41_015_625 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.95 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 410_156 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/perplexity50/sbatch_4b284bc4perplexity50val.sh b/perplexity50/sbatch_4b284bc4perplexity50val.sh new file mode 100644 index 0000000000000000000000000000000000000000..40b886caa29b621fcf8ee427c3a542180355f174 --- /dev/null +++ b/perplexity50/sbatch_4b284bc4perplexity50val.sh @@ -0,0 +1,168 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=40 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=4b284bc4perplexity50val +VARIANT_CKPT=checkpoints_4b284bc4perplexity50 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=$VARIANT_CKPT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train1b5.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=2 + +MICRO_BATCH_SIZE=2 +GRADIENT_ACCUMULATION_STEPS=2 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_4516M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 25611230000 +# -> Samples: 12505484 +TRAIN_SAMPLES=1 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 0 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + --override-lr-scheduler \ + --reset-progress \ + --no-load-optim \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1 \ + --eval-iters 100 \ + --eval-only true \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun_32.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/perplexity50/tensorboard_4b284bc4perplexity50/events.out.tfevents.1683712349.nid005934.69593.0 b/perplexity50/tensorboard_4b284bc4perplexity50/events.out.tfevents.1683712349.nid005934.69593.0 new file mode 100644 index 0000000000000000000000000000000000000000..1e0221423d62c11ddc3da04443a4e0d65315d34b --- /dev/null +++ b/perplexity50/tensorboard_4b284bc4perplexity50/events.out.tfevents.1683712349.nid005934.69593.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0cf7f1616f324a84b4e6c75c8992d6a3a44116f31565f45861a434638f82f39 +size 1067852 diff --git a/perplexity50/tensorboard_4b284bc4perplexity50/events.out.tfevents.1683886096.nid007164.93867.0 b/perplexity50/tensorboard_4b284bc4perplexity50/events.out.tfevents.1683886096.nid007164.93867.0 new file mode 100644 index 0000000000000000000000000000000000000000..4a7ffb13c924ec52babfc39488e98dac84de33ad --- /dev/null +++ b/perplexity50/tensorboard_4b284bc4perplexity50/events.out.tfevents.1683886096.nid007164.93867.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e5e5ef539c0c64bffd2bfeaf710778cd4a37194b445237a5094ccf6451b6f9a +size 36156602 diff --git a/perplexity50/tensorboard_4b284bc4perplexity50/events.out.tfevents.1683941929.nid005934.124810.0 b/perplexity50/tensorboard_4b284bc4perplexity50/events.out.tfevents.1683941929.nid005934.124810.0 new file mode 100644 index 0000000000000000000000000000000000000000..55d3de418d0a38ddd97884220ea411f5f1981e01 --- /dev/null +++ b/perplexity50/tensorboard_4b284bc4perplexity50/events.out.tfevents.1683941929.nid005934.124810.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6439097b3e220a960d5a6139871bc4dbab935f73d0f7db4a03f9a8bb4d00569c +size 16424 diff --git a/perplexity50/tensorboard_4b284bc4perplexity50val/events.out.tfevents.1683959866.nid006672.78091.0 b/perplexity50/tensorboard_4b284bc4perplexity50val/events.out.tfevents.1683959866.nid006672.78091.0 new file mode 100644 index 0000000000000000000000000000000000000000..0dd2efff080b2700e083fd3aae3af9db98d49300 --- /dev/null +++ b/perplexity50/tensorboard_4b284bc4perplexity50val/events.out.tfevents.1683959866.nid006672.78091.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:146650bb8c16992ceb839b6eacc17af0842ba2694e07ab89aeeaa9e29d8d341d +size 980 diff --git a/perplexity50/transformers/config.json b/perplexity50/transformers/config.json new file mode 100644 index 0000000000000000000000000000000000000000..df814c3a71ee49044410636b5eeb4f43611690ba --- /dev/null +++ b/perplexity50/transformers/config.json @@ -0,0 +1 @@ +{"vocab_size": 50432, "n_positions": 2048, "n_embd": 3072, "n_layer": 36, "n_head": 24, "n_inner": 12288, "activation_function": "gelu", "resid_pdrop": 0.1, "embd_pdrop": 0.1, "attn_pdrop": 0.1, "layer_norm_epsilon": 1e-05, "initializer_range": 0.02, "summary_type": "cls_index", "summary_use_proj": true, "summary_activation": null, "summary_first_dropout": 0.1, "summary_proj_to_labels": true, "scale_attn_weights": true, "use_cache": true, "scale_attn_by_inverse_layer_idx": false, "reorder_and_upcast_attn": false, "bos_token_id": 50256, "eos_token_id": 50256, "return_dict": true, "output_hidden_states": false, "output_attentions": false, "torchscript": false, "torch_dtype": null, "use_bfloat16": false, "tf_legacy_loss": false, "pruned_heads": {}, "tie_word_embeddings": true, "is_encoder_decoder": false, "is_decoder": false, "cross_attention_hidden_size": null, "add_cross_attention": false, "tie_encoder_decoder": false, "max_length": 20, "min_length": 0, "do_sample": false, "early_stopping": false, "num_beams": 1, "num_beam_groups": 1, "diversity_penalty": 0.0, "temperature": 1.0, "top_k": 50, "top_p": 1.0, "typical_p": 1.0, "repetition_penalty": 1.0, "length_penalty": 1.0, "no_repeat_ngram_size": 0, "encoder_no_repeat_ngram_size": 0, "bad_words_ids": null, "num_return_sequences": 1, "chunk_size_feed_forward": 0, "output_scores": false, "return_dict_in_generate": false, "forced_bos_token_id": null, "forced_eos_token_id": null, "remove_invalid_values": false, "exponential_decay_length_penalty": null, "suppress_tokens": null, "begin_suppress_tokens": null, "architectures": ["GPT2LMHeadModel"], "finetuning_task": null, "id2label": {"0": "LABEL_0", "1": "LABEL_1"}, "label2id": {"LABEL_0": 0, "LABEL_1": 1}, "tokenizer_class": null, "prefix": null, "pad_token_id": null, "sep_token_id": null, "decoder_start_token_id": null, "task_specific_params": null, "problem_type": null, "_name_or_path": "", "transformers_version": "4.25.0.dev0", "n_ctx": 1024, "gradient_checkpointing": false, "model_type": "gpt2"} \ No newline at end of file diff --git a/perplexity50/transformers/pytorch_model.bin b/perplexity50/transformers/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..359dac85df3856ca9b3f25b2a31fae821a23f8b7 --- /dev/null +++ b/perplexity50/transformers/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc9dff4d57adff68ab3468f974fdda5c0b9ab676dccbab4bec81892ceb8d626c +size 8781203669